RSS-83: More improvements for section detection 2

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java

@@ -10,6 +10,8 @@ import lombok.RequiredArgsConstructor;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+
 @Data
 @RequiredArgsConstructor
 public class Page {
@@ -32,6 +34,7 @@ public class Page {
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
 
     private double cropBoxArea;
+    private PDRectangle cropbox;
 
 
     public boolean isRotated() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -45,6 +45,7 @@ public class BlockificationService {
         for (TextPositionSequence word : textPositions) {
 
             boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
+            boolean lineSeparation2 = prev != null && minY - word.getY2() > word.getHeight() * 0.75 && word.getFontSize() != prev.getFontSize();
             boolean startFromTop = word.getY1() > maxY + word.getHeight();
             boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
             boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
@@ -53,7 +54,9 @@ public class BlockificationService {
                     .getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines) || word
                     .getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines);
 
-            if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
+            boolean isOtherFont = prev != null && word.getFontStyle() != prev.getFontStyle();
+
+            if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling || lineSeparation2 || isOtherFont)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -1,16 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;
 
+import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.springframework.stereotype.Service;
 
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
 import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 @Slf4j
@@ -32,12 +43,38 @@ public class ClassificationService {
 
         for (Page page : document.getPages()) {
             Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
+
+            if (!page.isLandscape() && page.getRotation() == 270){
+                btf = getRectangle(btf, page.getCropbox());
+            }
+            System.out.println("CropBoxArea" + page.getCropBoxArea());
             page.setBodyTextFrame(btf);
             classifyPage(btf, page, document, headlineFontSizes);
         }
     }
 
 
+
+    public Rectangle getRectangle(Rectangle rectangle, PDRectangle cropbox) {
+
+
+        Point2D bottomLeft = new Point2D.Double(rectangle.getMinX(), rectangle.getMinY());
+
+        AffineTransform transform = new AffineTransform();
+
+        float ty = (cropbox.getLowerLeftY() + cropbox.getUpperRightY()) / 2;
+
+        transform.translate(ty, ty);
+        transform.rotate(Math.toRadians(180), 0, 0);
+        transform.translate(-ty, -ty);
+
+        bottomLeft = transform.transform(bottomLeft, null);
+
+      return new Rectangle((float)bottomLeft.getX() - (float) rectangle.getHeight(),(float) rectangle.getX(),  (float) rectangle.getWidth(), (float) rectangle.getHeight());
+    }
+
+
+
     public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
 
         for (AbstractTextContainer textBlock : page.getTextBlocks()) {
@@ -51,11 +88,44 @@ public class ClassificationService {
     public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
                               List<Float> headlineFontSizes) {
 
-        if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification("Other");
-            return;
+        var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true);
+        var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
+
+        Matcher matcher = pattern.matcher(textBlock.toString());
+        Matcher matcher2 = pattern2.matcher(textBlock.toString());
+
+
+
+//        if (document.getFontSizeCounter().getMostPopular() == null) {
+//            textBlock.setClassification("Other");
+//            return;
+//        }
+
+//        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
+//                .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
+//                        .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
+//                .getMostPopular() || page.getTextBlocks().size() == 1)) {
+//            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
+//                textBlock.setClassification("Title");
+//            }
+
+            if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 5.9
+                    && !textBlock.getOrientation().equals(Orientation.RIGHT)
+
+                    && ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)))
+                         || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT))
+                         || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE"))) {
+                    textBlock.setClassification("H 1");
+                    document.setHeadlines(true);
+
         }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
+            else
+             if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) {
+                textBlock.setClassification("H 2");
+                document.setHeadlines(true);
+        }
+
+       else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
                 .getMostPopular())) {
             textBlock.setClassification("Header");
@@ -64,54 +134,36 @@ public class ClassificationService {
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
                 .getMostPopular())) {
             textBlock.setClassification("Footer");
-        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
-                .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
-                        .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
-                .getMostPopular() || page.getTextBlocks().size() == 1)) {
-            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification("Title");
         }
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
-                .getFontSizeCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle()
-                .equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document
-                .getFontSizeCounter()
-                .getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    textBlock.setClassification("H " + i);
-                    document.setHeadlines(true);
-                }
-            }
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
-                .startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
-            document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
-                .getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
-            textBlock.setClassification("TextBlock Bold");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
-                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter()
-                        .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular()) {
+//            else if (textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getMostPopular()
+//                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            document.setHeadlines(true);
+//        }
+//        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
+//                .getFontSizeCounter()
+//                .getMostPopular() && textBlock.getMostPopularWordStyle()
+//                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+//            textBlock.setClassification("TextBlock Bold");
+//        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
+//                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
+//                .equals(document.getFontStyleCounter()
+//                        .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+//                .getMostPopular()) {
+//            textBlock.setClassification("TextBlock");
+//        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
+//                .getFontSizeCounter()
+//                .getMostPopular() && textBlock.getMostPopularWordStyle()
+//                .equals("italic") && !document.getFontStyleCounter()
+//                .getMostPopular()
+//                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+//            textBlock.setClassification("TextBlock Italic");
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
         textBlock.setClassification("TextBlock");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
-                .getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
-            textBlock.setClassification("TextBlock Italic");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
-            textBlock.setClassification("TextBlock Unknown");
-        } else {
+    }
+        else {
             textBlock.setClassification("Other");
         }
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java

@@ -31,16 +31,16 @@ public class PositionUtils {
     }
 
 
-    public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, boolean rotated) {
+    public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock) {
 
         if (btf == null || textBlock == null) {
             return false;
         }
 
-        if (rotated && textBlock.getMinX() < btf.getX()) {
+        if ((textBlock.getRotation() == 0 || textBlock.getRotation() == 180)  && textBlock.getMinY() > btf.getY() + btf.getHeight()) {
             // Its very strange, P{0,0} is on top left in this case, instead of lower left.
             return true;
-        } else if (!rotated && textBlock.getMinY() > btf.getY() + btf.getHeight()) {
+        } else if ((textBlock.getRotation() == 90 || textBlock.getRotation() == 270) && textBlock.getMinX() < btf.getX()) {
             return true;
         } else {
             return false;
@@ -89,6 +89,9 @@ public class PositionUtils {
 
 
     public Float getApproxLineCount(TextBlock textBlock) {
+
+//        float height = textBlock.getRotation() == 0 || textBlock.getRotation() == 180 ? textBlock.getHeight() : textBlock.getWidth();
+
         return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -9,9 +9,12 @@ import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.SystemUtils;
@@ -29,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.classification.service.Classifi
 import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
 import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
@@ -89,24 +93,44 @@ public class PdfSegmentationService {
                     stripper.getText(pdDocument);
 
                     PDRectangle pdr = pdPage.getMediaBox();
-                    boolean isLandscape = pdr.getWidth() > pdr.getHeight();
-
                     int rotation = pdPage.getRotation();
+                    boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) ||
+                            pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+
+
+
+
+                    System.out.println("PageNr: " + pageNumber + " rotation: "+rotation);
+
                     boolean isRotated = rotation != 0 && rotation != 360;
 
                     CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
                             .getMaxCharHeight());
 
+//                    var sorted = stripper.getTextPositionSequences().stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
+
+
+
+
+
+
                     Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
                             .getVertical());
 
 
+
+
+
+//                    page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinX()));
+
                     PDRectangle cropbox = pdPage.getCropBox();
                     float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
                     page.setCropBoxArea(cropboxArea);
+                    page.setCropbox(cropbox);
 
                     page.setRotation(rotation);
-                    page.setLandscape(isLandscape || isRotated);
+                    page.setLandscape(isLandscape);
                     page.setPageNumber(pageNumber);
 
 
@@ -119,11 +143,34 @@ public class PdfSegmentationService {
                         imageService.findOcr(page);
                     }
 
+
+                    var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true);
+                    page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder()));
+
+                    AbstractTextContainer prev = null;
+                    List<AbstractTextContainer> toRemove = new ArrayList<>();
+                    for (AbstractTextContainer current : page.getTextBlocks()) {
+
+                        if(prev != null && prev instanceof TextBlock && current instanceof TextBlock && Math.abs(prev.getMinY() - current.getMinY()) <=1){
+                            Matcher matcher = pattern.matcher(prev.getText());
+                            if (matcher.matches()) {
+                                ((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0));
+                                toRemove.add(prev);
+                                current.setMinX(prev.getMinX());
+                            }
+                        }
+                        prev = current;
+                    }
+
+                    page.getTextBlocks().removeAll(toRemove);
+
+
                     pages.add(page);
                 }
 
                 document.setPages(pages);
 
+
                 classificationService.classifyDocument(document);
                 sectionsBuilderService.buildSections(document);
                 sectionsBuilderService.addImagesToSections(document);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -68,7 +68,7 @@ public class SectionsBuilderService {
                     continue;
                 }
 
-                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
+                if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -1,9 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.visualization.service;
 
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -16,6 +20,8 @@ import org.apache.pdfbox.pdmodel.font.PDType1Font;
 import org.springframework.stereotype.Service;
 
 import java.awt.Color;
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.List;
 
@@ -32,6 +38,7 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
+            int a = 0;
             for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
 
                 for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
@@ -43,7 +50,8 @@ public class PdfVisualisationService {
                     }
                     if (textBlock instanceof TextBlock) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
-                        visualizeTextBlock((TextBlock) textBlock, contentStream);
+                        visualizeTextBlock((TextBlock) textBlock, contentStream, a);
+                        a++;
                     } else if (textBlock instanceof Table) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
                         visualizeTable((Table) textBlock, contentStream);
@@ -59,7 +67,9 @@ public class PdfVisualisationService {
 
     public void visualizeClassifications(Document classifiedDoc, PDDocument document) throws IOException {
 
+
         for (int page = 1; page <= document.getNumberOfPages(); page++) {
+            int a = 0;
 
             Page analyzedPage = classifiedDoc.getPages().get(page - 1);
 
@@ -72,14 +82,20 @@ public class PdfVisualisationService {
                     continue;
                 }
                 if (textBlock instanceof TextBlock) {
-                    visualizeTextBlock((TextBlock) textBlock, contentStream);
+                    visualizeTextBlock((TextBlock) textBlock, contentStream, a);
+                    a++;
                 } else if (textBlock instanceof Table) {
                     visualizeTable((Table) textBlock, contentStream);
                 }
             }
 
+
             contentStream.setStrokingColor(Color.YELLOW);
-            contentStream.addRect((float) analyzedPage.getBodyTextFrame().getX(), (float) analyzedPage.getBodyTextFrame().getY(), (float) analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight());
+
+            var rect = getRectangle((float)  analyzedPage.getBodyTextFrame().getX(),(float) analyzedPage.getBodyTextFrame().getY(),(float)  analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight(), pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight());
+
+            contentStream.addRect(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
+
             contentStream.stroke();
 
             contentStream.close();
@@ -87,28 +103,64 @@ public class PdfVisualisationService {
     }
 
 
-    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
+    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream, int number) throws IOException {
 
         contentStream.setStrokingColor(Color.RED);
 
-        contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
+        var rect = getRectangle(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight(), textBlock.getRotation(), textBlock.getSequences().get(0).getPageWidth(), textBlock.getSequences().get(0).getPageHeight());
+
+        contentStream.addRect(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
         contentStream.stroke();
 
         if (textBlock.getClassification() != null) {
             contentStream.beginText();
 
             contentStream.setNonStrokingColor(Color.BLUE);
-            contentStream.setFont(PDType1Font.TIMES_ROMAN, 12f);
+            contentStream.setFont(PDType1Font.TIMES_ROMAN, 6f);
 
-            contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
+            contentStream.newLineAtOffset(rect.getTopLeft().getX(), rect.getTopLeft().getY() + rect.getHeight());
 
-            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation());
+            contentStream.showText(textBlock.getClassification() /*" textBlock.getOrientation() + "--> "*/ + number);
 
             contentStream.endText();
         }
     }
 
 
+
+    public Rectangle getRectangle(float x, float y, float width, float height, int rotation, float pageWidth, float pageHeight) {
+
+
+        if(rotation == 90){
+            return new Rectangle( //
+                    new Point(x, y), width, height, 0);
+        }
+
+        Point2D bottomLeft = new Point2D.Double(x, pageHeight - y);
+        Point2D topRight = new Point2D.Double(x + width,pageHeight - y - height);
+
+        AffineTransform transform = new AffineTransform();
+        if (rotation == 0 || rotation == 180) {
+            transform.rotate(Math.toRadians(rotation), pageWidth / 2f, pageHeight / 2f);
+            transform.translate(0f, pageHeight);
+            transform.scale(1., -1.);
+        }  else {
+            transform.rotate(Math.toRadians(rotation), pageHeight / 2f, pageHeight / 2f);
+            transform.translate(0f, pageWidth);
+            transform.scale(1., -1.);
+        }
+
+        bottomLeft = transform.transform(bottomLeft, null);
+        topRight = transform.transform(topRight, null);
+
+        return new Rectangle( //
+                new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), 0);
+    }
+
+
+
+
+
     private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
         for (List<Cell> row : table.getRows()) {
             for (Cell cell : row) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -334,7 +334,9 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/32 - Emamectin Benzoate Technical - Acute Oral Toxicity - Mouse.pdf");
+        System.out.println("Detailed quantitative and qualitative information on the composition of the plant ".length());
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
@@ -1094,7 +1096,7 @@ public class RedactionIntegrationTest {
     public void classificationTest() throws IOException {
 
         System.out.println("classificationTest");
-        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/02 - A22833B - Acute Oral (Up and Down) - Final Report.pdf");
 
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -15,379 +15,9 @@ global Section section
 //    end
 
 
-rule "0: Add CBI_author from ai"
+ rule "1: Find headlines "
      when
-        Section(aiMatchesType("CBI_author"))
+         Section(text.length() > 1)
      then
-        section.addAiEntities("CBI_author", "CBI_author");
-    end
-
-rule "0: Combine ai types CBI_author from ai"
-    when
-        Section(aiMatchesType("ORG"))
-    then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
-    end
-
-rule "0: Expand CBI Authors with firstname initials"
-    when
-        Section(matchesType("CBI_author"))
-    then
-        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
-    end
-
-rule "0: Expand CBI_author and PII matches with salutation prefix"
-    when
-        Section((matchesType("CBI_author") || matchesType("PII")) && (
-            searchText.contains("Mr")
-            || searchText.contains("Mrs")
-            || searchText.contains("Ms")
-            || searchText.contains("Miss")
-            || searchText.contains("Sir")
-            || searchText.contains("Madam")
-            || searchText.contains("Madame")
-            || searchText.contains("Mme")
-        ))
-    then
-        section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
-        section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
-    end
-
-
-rule "1: Redacted because Section contains Vertebrate"
-    when
-        Section(matchesType("vertebrate"))
-    then
-        section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "2: Not Redacted because Section contains no Vertebrate"
-    when
-        Section(!matchesType("vertebrate"))
-    then
-        section.redactNot("CBI_author", 2, "No Vertebrate found");
-        section.redactNot("CBI_address", 2, "No Vertebrate found");
-    end
-
-
-rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
-    then
-        section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
-        section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
-    end
-
-
-rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
-    then
-        section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "5: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("published_information"))
-    then
-        section.redactNotAndReference("CBI_author","published_information", 5, "Vertebrate and Published Information found");
-        section.redactNotAndReference("CBI_address","published_information", 5, "Vertebrate and Published Information found");
-    end
-
-
-rule "6: Not redacted because Vertebrate Study = N"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
-    then
-        section.redactNotCell("Author(s)", 6, "CBI_author", true, "Not redacted because row is not a vertebrate study");
-        section.redactNot("CBI_author", 6, "Not redacted because row is not a vertebrate study");
-        section.redactNot("CBI_address", 6, "Not redacted because row is not a vertebrate study");
-        section.highlightCell("Vertebrate study Y/N", 6, "hint_only");
-    end
-
-
-rule "7: Redact if must redact entry is found"
-    when
-        Section(matchesType("must_redact"))
-    then
-        section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
-    then
-        section.redactCell("Author(s)", 8, "CBI_author", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 8, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.highlightCell("Vertebrate study Y/N", 8, "must_redact");
-    end
-
-
-rule "9: Redact sponsor company"
-    when
-        Section(searchText.toLowerCase().contains("batches produced at"))
-    then
-        section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 9, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("batches produced at", "must_redact");
-    end
-
-
-rule "10: Redact determination of residues"
-    when
-        Section((
-            searchText.toLowerCase.contains("determination of residues") ||
-            searchText.toLowerCase.contains("determination of total residues")
-        ) && (
-            searchText.toLowerCase.contains("livestock") ||
-            searchText.toLowerCase.contains("live stock") ||
-            searchText.toLowerCase.contains("tissue") ||
-            searchText.toLowerCase.contains("tissues") ||
-            searchText.toLowerCase.contains("liver") ||
-            searchText.toLowerCase.contains("muscle") ||
-            searchText.toLowerCase.contains("bovine") ||
-            searchText.toLowerCase.contains("ruminant") ||
-            searchText.toLowerCase.contains("ruminants")
-        ))
-    then
-        section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("determination of residues", "must_redact");
-        section.addHintAnnotation("livestock", "must_redact");
-        section.addHintAnnotation("live stock", "must_redact");
-        section.addHintAnnotation("tissue", "must_redact");
-        section.addHintAnnotation("tissues", "must_redact");
-        section.addHintAnnotation("liver", "must_redact");
-        section.addHintAnnotation("muscle", "must_redact");
-        section.addHintAnnotation("bovine", "must_redact");
-        section.addHintAnnotation("ruminant", "must_redact");
-        section.addHintAnnotation("ruminants", "must_redact");
-    end
-
-
-rule "11: Redact if CTL/* or BL/*  was found"
-    when
-        Section(searchText.contains("CTL/") || searchText.contains("BL/"))
-    then
-        section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("CTL", "must_redact");
-        section.addHintAnnotation("BL", "must_redact");
-    end
-
-
-rule "12: Redact and add recommendation for et al. author"
-    when
-        Section(searchText.contains("et al"))
-    then
-		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "13: Add recommendation for Addresses in Test Organism sections"
-    when
-        Section(searchText.contains("Species:") && searchText.contains("Source:"))
-    then
-		section.recommendLineAfter("Source:", "CBI_address");
-    end
-
-
-rule "14: Add recommendation for Addresses in Test Animals sections"
-    when
-        Section(searchText.contains("Species") && searchText.contains("Source"))
-    then
-		section.recommendLineAfter("Source", "CBI_address");
-    end
-
-// --------------------------------------- PII rules -------------------------------------------------------------------
-
-
-rule "14: Redacted PII Personal Identification Information"
-    when
-        Section(matchesType("PII"))
-    then
-        section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "15: Redact Emails by RegEx"
-    when
-        Section(searchText.contains("@"))
-    then
-        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "16: Redact contact information"
-    when
-       Section(text.contains("Contact point:")
-       || text.contains("Phone:")
-       || text.contains("Fax:")
-       || text.contains("Tel.:")
-       || text.contains("Tel:")
-       || text.contains("E-mail:")
-       || text.contains("Email:")
-       || text.contains("e-mail:")
-       || text.contains("E-mail address:")
-       || text.contains("Alternative contact:")
-       || text.contains("Telephone number:")
-       || text.contains("Telephone No:")
-       || text.contains("Fax number:")
-       || text.contains("Telephone:")
-       || text.contains("European contact:"))
-    then
-          section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("European contact:",  "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-     end
-
-
-rule "17: Redact contact information if applicant is found"
-    when
-        Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
-    then
-        section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("European contact:",  "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "18: Redact contact information if Producer is found"
-    when
-        Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
-    then
-        section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "19: Redact AUTHOR(S)"
-    when
-        Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true"))
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "20: Redact PERFORMING LABORATORY"
-    when
-        Section(searchText.contains("PERFORMING LABORATORY:"))
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "21: Redact On behalf of Sequani Ltd.:"
-    when
-        Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
-    then
-        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "22: Redact On behalf of Syngenta Ltd.:"
-    when
-        Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
-    then
-        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-// --------------------------------------- other rules -------------------------------------------------------------------
-
-rule "25: Redact Purity"
-    when
-        Section(searchText.contains("purity"))
-    then
-	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
-    end
-
-
-rule "26: Redact signatures"
-    when
-        Section(matchesImageType("signature"))
-    then
-        section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "27: Redact formula"
-    when
-        Section(matchesImageType("formula"))
-    then
-        section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "28: Redact Logos"
-    when
-        Section(matchesImageType("logo"))
-    then
-        section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "29: Redact Dossier Redactions"
-     when
-         Section(matchesType("dossier_redactions"))
-     then
-         section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
-     end
-
-rule "30: Ignore dossier_redactions if confidential"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
-    then
-        section.ignore("dossier_redactions");
-    end
-
-// ex. "New Rules for PAD" - "Annex A" - page 21, page 35 (table without header), page 38 (in-text)
-// https://www.regexplanet.com/share/index.html?share=yyyypb71xkr
-rule "101: Redact CAS numbers"
-    when
-        Section(hasTableHeader("Sample #"))
-    then
-        section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+         section.redactHeadline("PII", 1, "Headline found", "n-a.");
      end