Pull request #124 : Adjusted rules to new requirements, add possibility to addRedaction for rules

Merge in RED/redaction-service from NewRequirementsRules to master * commit 'b6a13d1ff8a55f1b5ef528af5ea726b5e17b4950': Adjusted rules to new requirements, add possibility to addRedaction for rules
Adjusted rules to new requirements, add possibility to addRedaction for rules
2021-02-18 10:48:26 +01:00 · 2021-02-18 10:33:38 +01:00 · 2021-02-18 09:18:59 +01:00 · 2021-02-18 08:58:51 +01:00 · 2021-02-12 10:55:04 +01:00 · 2021-02-12 10:41:32 +01:00
31 changed files with 734 additions and 289 deletions
@@ -72,14 +72,6 @@ public class PlanSpec {
                        .jobs(new Job("Default Job",
                                new BambooKey("JOB1"))
                                .tasks(
-                                        new ScriptTask()
-                                                .description("Clean")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-                                                        "rm -rf ./*"),
-                                        new VcsCheckoutTask()
-                                                .description("Checkout Default Repository")
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
                                        new ScriptTask()
                                                .description("Clean")
                                                .inlineBody("#!/bin/bash\n" +
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.0.2</version>
+        <version>1.0.3</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class ManualForceRedact {
+
+    private String id;
+    private String user;
+    private Status status;
+    private String legalBasis;
+
+}
@@ -1,5 +1,5 @@
 package com.iqser.red.service.redaction.v1.model;

 public enum ManualRedactionType {
-    ADD, REMOVE
+    ADD, REMOVE, FORCE_REDACT
 }
@@ -20,6 +20,9 @@ public class ManualRedactions {
    @Builder.Default
    private Set<IdRemoval> idsToRemove = new HashSet<>();

+    @Builder.Default
+    private Set<ManualForceRedact> forceRedacts = new HashSet<>();
+
    @Builder.Default
    private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();

@@ -18,6 +18,9 @@ public class Document {

    private List<Page> pages = new ArrayList<>();
    private List<Paragraph> paragraphs = new ArrayList<>();
+    private List<Header> headers = new ArrayList<>();
+    private List<Footer> footers = new ArrayList<>();
+    private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
    private Map<Integer, List<Entity>> entities = new HashMap<>();
    private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
    private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class Footer {
+
+    private List<TextBlock> textBlocks;
+
+
+    public SearchableText getSearchableText() {
+
+        SearchableText searchableText = new SearchableText();
+        textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
+        return searchableText;
+    }
+
+}
@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class Header {
+
+    private List<TextBlock> textBlocks;
+
+
+    public SearchableText getSearchableText() {
+
+        SearchableText searchableText = new SearchableText();
+        textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
+        return searchableText;
+    }
+
+}
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;

+import java.awt.geom.Rectangle2D;
 import java.util.List;

 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@@ -16,6 +17,8 @@ public class Page {
    @NonNull
    private List<AbstractTextContainer> textBlocks;

+    private List<Rectangle2D> imageBounds;
+
    private Rectangle bodyTextFrame;

    private boolean landscape;
@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class UnclassifiedText {
+
+    private List<TextBlock> textBlocks;
+
+
+    public SearchableText getSearchableText() {
+
+        SearchableText searchableText = new SearchableText();
+        textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
+        return searchableText;
+    }
+
+}
@@ -111,6 +111,8 @@ public class ClassificationService {
            textBlock.setClassification("TextBlock Italic");
        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
            textBlock.setClassification("TextBlock Unknown");
+        } else {
+            textBlock.setClassification("Other");
        }
    }

@@ -64,8 +64,12 @@ public class RedactionController implements RedactionResource {

            log.info("Redaction analysis successful...");

-            return AnalyzeResult.builder().sectionGrid(classifiedDoc.getSectionGrid()).redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
-                    .getRulesVersion(), analyzeRequest.getRuleSetId())).build();
+            return AnalyzeResult.builder()
+                    .sectionGrid(classifiedDoc.getSectionGrid())
+                    .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
+                    .getRulesVersion(), analyzeRequest.getRuleSetId()))
+                    .numberOfPages(classifiedDoc.getPages().size())
+                    .build();

        } catch (Exception e) {
            throw new RedactionException(e);
@@ -1,6 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.parsing;

+import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -28,10 +30,15 @@ import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
 import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
+import org.apache.pdfbox.cos.COSName;
 import org.apache.pdfbox.cos.COSNumber;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.graphics.PDXObject;
+import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
 import org.apache.pdfbox.text.PDFTextStripper;
 import org.apache.pdfbox.text.TextPosition;
+import org.apache.pdfbox.util.Matrix;

 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
@@ -43,8 +50,17 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
 public class PDFLinesTextStripper extends PDFTextStripper {

+    @Setter
+    protected PDPage pdpage;
+
    @Getter
-    private int maxCharWidths;
+    private int minCharWidth;
+
+    @Getter
+    private int maxCharWidth;
+    
+    @Getter
+    private int minCharHeight;

    @Getter
    private int maxCharHeight;
@@ -57,13 +73,18 @@ public class PDFLinesTextStripper extends PDFTextStripper {

    private final List<Ruling> graphicsPath = new ArrayList<>();

+    @Getter
+    private List<Rectangle2D> imageBounds = new ArrayList<>();
+
    private float path_x;
    private float path_y;

    @Setter
    private int pageNumber;

+
    public PDFLinesTextStripper() throws IOException {
+
        super();
        this.addOperator(new SetStrokingColorSpace());
        this.addOperator(new SetNonStrokingColorSpace());
@@ -87,9 +108,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        this.addOperator(new SetLineWidth());
    }

+
    @Override
-    protected void processOperator(Operator operator, List<COSBase> arguments)
-            throws IOException {
+    protected void processOperator(Operator operator, List<COSBase> arguments) throws IOException {

        String operation = operator.getName();

@@ -110,9 +131,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {

                    // The direction of vertical lines must always be from bottom to top for the table extraction algorithm.
                    if (pos.getY() > path_y) {
-                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos
+                                .getY())));
                    } else {
-                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos.getX(), path_y)));
+                        graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos
+                                .getX(), path_y)));
                    }

                    path_x = (float) pos.getX();
@@ -133,19 +156,25 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                    Point2D p2 = transformPosition(x + width, y + height);

                    // Horizontal lines
-                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
-                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
+                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2
+                            .getX(), (float) p1.getY())));
+                    graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2
+                            .getX(), (float) p2.getY())));

                    // Vertical lines, direction must always be from bottom to top for the table extraction algorithm.
                    if (p2.getY() > p1.getY()) {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2
+                                .getX(), (float) p2.getY())));
                    } else {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2
+                                .getX(), (float) p1.getY())));
                    }
                    if (p2.getY() > p1.getY()) {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1.getX(), (float) p2.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1
+                                .getX(), (float) p2.getY())));
                    } else {
-                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1.getX(), (float) p1.getY())));
+                        graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1
+                                .getX(), (float) p1.getY())));
                    }
                }
                break;
@@ -168,12 +197,59 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            case OperatorName.ENDPATH:
                graphicsPath.clear();
                break;
+
+            case OperatorName.DRAW_OBJECT:
+                processImageOperation(arguments);
+                break;
+
        }

        super.processOperator(operator, arguments);
    }

+
+    protected void processImageOperation(List<COSBase> arguments) {
+
+        try {
+            COSName objectName = (COSName) arguments.get(0);
+            PDXObject xobject = getResources().getXObject(objectName);
+            if (xobject instanceof PDImageXObject) {
+                PDImageXObject pdfImage = (PDImageXObject) xobject;
+
+                Rectangle2D imageBounds = calculateImagePosition(pdfImage);
+
+                Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
+                        .getWidth(), (float) imageBounds.getHeight());
+
+                if (rect.getHeight() > 2 && rect.getWidth() > 2) {
+                    this.imageBounds.add(rect);
+                }
+            }
+        } catch (Exception e) {
+            log.warn("Problem during image extraction: {}", e.getMessage());
+        }
+    }
+
+
+    private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException {
+
+        Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
+
+        Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds();
+
+        AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform());
+        imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight());
+        imageTransform.translate(0, -pdfImage.getHeight());
+
+        AffineTransform pageTransform = new AffineTransform();
+        pageTransform.concatenate(imageTransform);
+
+        return pageTransform.createTransformedShape(imageBounds).getBounds2D();
+    }
+
+
    private float floatValue(COSBase value) {
+
        if (value instanceof COSNumber) {
            return ((COSNumber) value).floatValue();
        } else {
@@ -181,38 +257,56 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        }
    }

+
    private Point2D.Float transformPosition(float x, float y) {
+
        return super.transformedPoint(x, y);
    }

+
    private void addVisibleRulings(List<Ruling> path, boolean stroke) throws IOException {

        try {
-            if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor().toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
+            if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor()
+                    .toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor()
+                    .isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
                rulings.addAll(path);
            }
        } catch (UnsupportedOperationException e) {
-            log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor().getColorSpace().getName() + " or " + getGraphicsState().getNonStrokingColor().getColorSpace().getName() + " does not support toRGB");
+            log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor()
+                    .getColorSpace()
+                    .getName() + " or " + getGraphicsState().getNonStrokingColor()
+                    .getColorSpace()
+                    .getName() + " does not support toRGB");
        }
    }

+
    @Override
    public void writeString(String text, List<TextPosition> textPositions) throws IOException {

        int startIndex = 0;
        for (int i = 0; i <= textPositions.size() - 1; i++) {

+            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
+            if (charWidth < minCharWidth) {
+                minCharWidth = charWidth;
+            }
+            if (charWidth > maxCharWidth) {
+                maxCharWidth = charWidth;
+            }
+
            int charHeight = (int) textPositions.get(i).getHeightDir();
-            if(charHeight > maxCharHeight){
+            if (charHeight < minCharHeight) {
+                minCharHeight = charHeight;
+            }
+            if (charWidth > maxCharHeight) {
                maxCharHeight = charHeight;
            }

-            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
-            if(charWidth > maxCharWidths){
-                maxCharWidths = charWidth;
-            }
-
-            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
+            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
+                    .getUnicode()
+                    .equals("\u00A0"))) {
                startIndex++;
                continue;
            }
@@ -220,15 +314,21 @@ public class PDFLinesTextStripper extends PDFTextStripper {
            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
            if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
                }
                startIndex = i;
            }

-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
+            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
+                    .getUnicode()
+                    .equals("\u00A0")) && i <= textPositions.size() - 2) {
                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
+                        .getUnicode()
+                        .equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
                }
                startIndex = i + 1;
@@ -236,21 +336,29 @@ public class PDFLinesTextStripper extends PDFTextStripper {
        }

        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
+        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
+                .getUnicode()
+                .equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
            sublist = sublist.subList(0, sublist.size() - 1);
        }
-        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
+        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
+                .getUnicode()
+                .equals("\u00A0")))) {
            textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
        }
        super.writeString(text);
    }

+
    @Override
    public String getText(PDDocument doc) throws IOException {

-        maxCharWidths = 0;
-        maxCharWidths = 0;
+        minCharWidth = Integer.MAX_VALUE;
+        maxCharWidth = 0;
+        minCharHeight = Integer.MAX_VALUE;
+        maxCharHeight = 0;
        textPositionSequences.clear();
+        imageBounds = new ArrayList<>();
        rulings.clear();
        graphicsPath.clear();
        path_x = 0.0f;
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.parsing.model;

+import java.awt.geom.Rectangle2D;
 import java.util.List;

 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
@@ -13,10 +14,11 @@ public class ParsedElements {

    private List<TextPositionSequence> sequences;
    private List<Ruling> rulings;
+    private List<Rectangle2D> imageBounds;

    private boolean landscape;
    private boolean rotated;

+    private float minCharWidth;
    private float maxCharWidth;
-    private float maxCharHeight;
 }
@@ -32,6 +32,7 @@ public class SearchableText {

    }

+
    @SuppressWarnings("checkstyle:ModifiedControlVariable")
    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive,
                                                     List<TextPositionSequence> sequencesSubList) {
@@ -66,9 +67,12 @@ public class SearchableText {
            for (int j = 0; j < searchSpace.get(i).length(); j++) {

                if (i > 0 && j == 0 && searchSpace.get(i).charAt(0, caseInsensitive) == ' ' && searchSpace.get(i - 1)
-                        .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
-                        .charAt(j, caseInsensitive) == ' ' && searchSpace.get(i).charAt(j - 1, caseInsensitive) == ' ') {
-                    if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
+                        .charAt(searchSpace.get(i - 1)
+                                .length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
+                        .charAt(j, caseInsensitive) == ' ' && searchSpace.get(i)
+                        .charAt(j - 1, caseInsensitive) == ' ') {
+                    if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions()
+                            .isEmpty()) {
                        crossSequenceParts.add(partMatch);
                    }
                    continue;
@@ -80,8 +84,8 @@ public class SearchableText {
                    counter++;
                }

-                if (searchSpace.get(i)
-                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i)
+                if (searchSpace.get(i).charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace
+                        .get(i)
                        .charAt(j, caseInsensitive) == '-') {

                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
@@ -100,14 +104,15 @@ public class SearchableText {
                    if (counter == searchString.length()) {
                        crossSequenceParts.add(partMatch);

-                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
-                                .length() - 1 && isSeparator(searchSpace.get(i)
+                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i)
+                                .length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i)
                                .charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
                                .length() - 1 && isSeparator(searchSpace.get(i + 1)
-                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
+                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i)
+                                .length() - 1 && searchSpace.get(i)
                                .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
                                .charAt(0, caseInsensitive) != ' ') {
-                            finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
+                            finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts, normalizedSearchString));
                        }

                        counter = 0;
@@ -130,15 +135,21 @@ public class SearchableText {
        }

        return finalMatches;
-
    }


-    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
+    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts,
+                                                                     String searchString) {
+
+        List<EntityPositionSequence> result = new ArrayList<>();
+        String asString = buildString(crossSequenceParts);
+        if (!asString.equalsIgnoreCase(searchString)) {
+            return result;
+        }

        String plainId = IdBuilder.buildId(crossSequenceParts);
        String id = plainId;
-        List<EntityPositionSequence> result = new ArrayList<>();
+
        int currentPage = -1;
        int idDiffentPageSuffix = 1;
        EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
@@ -173,6 +184,12 @@ public class SearchableText {
    @Override
    public String toString() {

+        return buildString(sequences);
+    }
+
+
+    public String buildString(List<TextPositionSequence> sequences) {
+
        StringBuilder sb = new StringBuilder();

        TextPositionSequence previous = null;
@@ -62,9 +62,10 @@ public class Section {
    }


-    public boolean hasTableHeader(String headerName){
-       String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
-       return tabularData != null && tabularData.containsKey(cleanHeaderName);
+    public boolean hasTableHeader(String headerName) {
+
+        String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
+        return tabularData != null && tabularData.containsKey(cleanHeaderName);
    }


@@ -80,6 +81,34 @@ public class Section {
    }


+    public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (Entity entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
+                continue;
+            }
+
+            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+
+            while (matcher.find()) {
+                String match = matcher.group(group);
+                if (StringUtils.isNotBlank(match)) {
+                    expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
+                            .getRedactionReason(), entity.getLegalBasis()));
+
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
+
    public void redact(String type, int ruleNumber, String reason, String legalBasis) {

        boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
@@ -111,6 +140,48 @@ public class Section {
    }


+    public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group, String asType) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (Entity entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
+                continue;
+            }
+
+            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+
+            while (matcher.find()) {
+                String match = matcher.group(group);
+                if (StringUtils.isNotBlank(match)) {
+                    expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null));
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
+
+    public void addHintAnnotationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Matcher matcher = compiledPattern.matcher(searchText);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match)) {
+                Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null);
+                EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+            }
+        }
+    }
+
+
    public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason, String legalBasis) {

        entities.forEach(entity -> {
@@ -127,7 +198,14 @@ public class Section {
    public void addHintAnnotation(String value, String asType) {

        Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null);
-        EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+        EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
+    }
+
+
+    public void addRedaction(String value, String asType, int ruleNumber, String reason, String legalBasis) {
+
+        Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis);
+        EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
    }


@@ -160,7 +238,7 @@ public class Section {

                String trimmedValue = value.trim();
                String cleanValue;
-                if(trimmedValue.startsWith(":")){
+                if (trimmedValue.startsWith(":")) {
                    cleanValue = trimmedValue.substring(1).trim();
                } else {
                    cleanValue = trimmedValue;
@@ -207,8 +285,8 @@ public class Section {
    }


-    public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
-                              String reason, String legalBasis) {
+    public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
+                                          int ruleNumber, String reason, String legalBasis) {

        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);

@@ -363,6 +441,7 @@ public class Section {
            }
        }
    }
+
 }


@@ -53,25 +53,24 @@ public class AnnotationService {

            List<RedactionLogEntry> logEntries = redactionLogPerPage.get(page);
            if (logEntries != null && !logEntries.isEmpty()) {
-                addAnnotations(logEntries, pdPage, page, redactionLog.getRuleSetId());
+                addAnnotations(logEntries, pdPage, page);
            }
        }
    }


-    private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page,
-                                String ruleSetId) throws IOException {
+    private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page) throws IOException {

        List<PDAnnotation> annotations = pdPage.getAnnotations();

        for (RedactionLogEntry entry : logEntries) {
-            annotations.addAll(createAnnotation(entry, page, ruleSetId, pdPage.getMediaBox(), pdPage.getCropBox()));
+            annotations.addAll(createAnnotation(entry, page, pdPage.getMediaBox(), pdPage.getCropBox()));
        }
    }


-    private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, String ruleSetId,
-                                                PDRectangle mediaBox, PDRectangle cropBox) {
+    private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, PDRectangle mediaBox,
+                                                PDRectangle cropBox) {

        List<PDAnnotation> annotations = new ArrayList<>();

@@ -89,7 +88,7 @@ public class AnnotationService {
        PDRectangle pdRectangle = toPDRectangle(rectangles, mediaBox, cropBox);
        annotation.setRectangle(pdRectangle);
        annotation.setQuadPoints(toQuadPoints(rectangles, mediaBox, cropBox));
-        if (!dictionaryService.isHint(redactionLogEntry.getType(), ruleSetId)) {
+        if (!redactionLogEntry.isHint()) {
            annotation.setContents(createAnnotationContent(redactionLogEntry));
        }
        annotation.setTitlePopup(redactionLogEntry.getId());
@@ -19,8 +19,11 @@ import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
@@ -113,7 +116,23 @@ public class EntityRedactionService {
                }
                sectionNumber.incrementAndGet();
            }
-            sectionSearchableTextPairs.add(processText(paragraph, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionSearchableTextPairs.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph
+                    .getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionNumber.incrementAndGet();
+        }
+
+        for (Header header : classifiedDoc.getHeaders()) {
+            sectionSearchableTextPairs.add(processText(header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionNumber.incrementAndGet();
+        }
+
+        for (Footer footer : classifiedDoc.getFooters()) {
+            sectionSearchableTextPairs.add(processText(footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
+            sectionNumber.incrementAndGet();
+        }
+
+        for (UnclassifiedText unclassifiedText : classifiedDoc.getUnclassifiedTexts()) {
+            sectionSearchableTextPairs.add(processText(unclassifiedText.getSearchableText(), unclassifiedText.getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
            sectionNumber.incrementAndGet();
        }

@@ -157,7 +176,6 @@ public class EntityRedactionService {

        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();

-        int rowNumber = 0;
        for (List<Cell> row : table.getRows()) {
            SearchableText searchableRow = new SearchableText();
            Map<String, CellValue> tabularData = new HashMap<>();
@@ -170,11 +188,14 @@ public class EntityRedactionService {
                addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
                int cellStart = start;

-                if(rowNumber != 0) {
+                if (!cell.isHeaderCell()) {
                    cell.getHeaderCells().forEach(headerCell -> {
                        StringBuilder headerBuilder = new StringBuilder();
                        headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
-                        String headerName = headerBuilder.toString().replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
+                        String headerName = headerBuilder.toString()
+                                .replaceAll("\n", "")
+                                .replaceAll(" ", "")
+                                .replaceAll("-", "");
                        tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
                    });
                }
@@ -205,8 +226,8 @@ public class EntityRedactionService {
                    .build(), searchableRow));

            sectionNumber.incrementAndGet();
-            rowNumber++;
        }
+
        return sectionSearchableTextPairs;
    }

@@ -251,14 +272,13 @@ public class EntityRedactionService {
    }


-    private SectionSearchableTextPair processText(Paragraph paragraph, ManualRedactions manualRedactions,
+    private SectionSearchableTextPair processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks,
+                                                  String headline, ManualRedactions manualRedactions,
                                                  AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber) {

-        SearchableText searchableText = paragraph.getSearchableText();
-        addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber
-                .intValue());
-        Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber.intValue(), dictionary, local);
+        addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue());
+        Set<Entity> entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local);
        surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);

        return new SectionSearchableTextPair(Section.builder()
@@ -269,7 +289,7 @@ public class EntityRedactionService {
                        .collect(Collectors.toSet()) : entities)
                .text(searchableText.getAsStringWithLinebreaks())
                .searchText(searchableText.toString())
-                .headline(paragraph.getHeadline())
+                .headline(headline)
                .sectionNumber(sectionNumber.intValue())
                .searchableText(searchableText)
                .dictionary(dictionary)
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -13,6 +14,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Comment;
 import com.iqser.red.service.redaction.v1.model.IdRemoval;
+import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
@@ -27,6 +29,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -37,6 +40,8 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class RedactionLogCreatorService {

+    private static final String IMAGE = "image";
+
    private final DictionaryService dictionaryService;


@@ -56,6 +61,30 @@ public class RedactionLogCreatorService {
            if (manualRedactionPages.contains(page)) {
                addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
            }
+
+            if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
+                addImageEntries(classifiedDoc, page, ruleSetId);
+            }
+        }
+    }
+
+
+    private void addImageEntries(Document classifiedDoc, int pageNumber, String ruleSetId) {
+
+        for (Rectangle2D imageBounds : classifiedDoc.getPages().get(pageNumber - 1).getImageBounds()) {
+            RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
+                    .id(IdBuilder.buildId(imageBounds, pageNumber))
+                    .color(getColor(IMAGE, ruleSetId))
+                    .type(IMAGE)
+                    .redacted(false)
+                    .isHint(true)
+                    .manual(false)
+                    .isDictionaryEntry(false)
+                    .isRecommendation(false)
+                    .positions(List.of(new Rectangle(new Point((float) imageBounds.getX(), (float) imageBounds.getY()), (float) imageBounds
+                            .getWidth(), (float) imageBounds.getHeight(), pageNumber)))
+                    .build();
+            classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
        }
    }

@@ -108,9 +137,11 @@ public class RedactionLogCreatorService {
                                redactionLogEntry.setRedacted(false);
                                redactionLogEntry.setStatus(Status.APPROVED);
                                manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
                            } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
                                manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
                                redactionLogEntry.setStatus(Status.REQUESTED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
                            } else {
                                redactionLogEntry.setStatus(Status.DECLINED);
                            }
@@ -121,7 +152,34 @@ public class RedactionLogCreatorService {
                            redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
                        }
                    }
+                }

+                if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
+                    for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
+                        if (manualForceRedact.getId().equals(entityPositionSequence.getId())) {
+                            String manualOverrideReason = null;
+                            if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
+                                entity.setRedaction(true);
+                                redactionLogEntry.setRedacted(true);
+                                redactionLogEntry.setStatus(Status.APPROVED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
+                                manualOverrideReason = entity.getRedactionReason() + ", forced by manual override";
+                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                            } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
+                                manualOverrideReason = entity.getRedactionReason() + ", requested to force redact";
+                                redactionLogEntry.setStatus(Status.REQUESTED);
+                                redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
+                                redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+                            } else {
+                                redactionLogEntry.setStatus(Status.DECLINED);
+                            }
+
+                            entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
+                            redactionLogEntry.setReason(manualOverrideReason);
+                            redactionLogEntry.setManual(true);
+                            redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
+                        }
+                    }
                }

                if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
@@ -166,7 +224,7 @@ public class RedactionLogCreatorService {
                    startIndex = i;
                }
            }
-            if (startIndex != textPositions.size() - 1) {
+            if (startIndex != textPositions.size()) {
                rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
            }
        }
@@ -121,4 +121,10 @@ public class EntitySearchUtils {
        }
        entities.add(found);
    }
+
+    public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found){
+        // HashSet keeps old value but we want the new.
+        entities.removeAll(found);
+        entities.addAll(found);
+    }
 }
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import java.awt.geom.Rectangle2D;
 import java.nio.charset.StandardCharsets;
 import java.util.List;

@@ -23,4 +24,16 @@ public class IdBuilder {

        return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
    }
+
+
+    public String buildId(Rectangle2D rectangle2D, int page){
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("x").append(rectangle2D.getX()).append("y").append(rectangle2D.getY()).append("h").append(rectangle2D.getHeight()).append("w").append(rectangle2D.getWidth()).append("p").append(page);
+
+        return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
+    }
+
+
+
 }
@@ -48,6 +48,7 @@ public class PdfSegmentationService {
            stripper.setPageNumber(pageNumber);
            stripper.setStartPage(pageNumber);
            stripper.setEndPage(pageNumber);
+            stripper.setPdpage(pdPage);
            stripper.getText(pdDocument);

            PDRectangle pdr = pdPage.getMediaBox();
@@ -56,18 +57,17 @@ public class PdfSegmentationService {
            int rotation = pdPage.getRotation();
            boolean isRotated = rotation != 0 && rotation != 360;

-
            ParsedElements parsedElements = ParsedElements.builder()
                    .rulings(stripper.getRulings())
                    .sequences(stripper.getTextPositionSequences())
-                    .maxCharWidth(stripper.getMaxCharWidths())
-                    .maxCharHeight(stripper.getMaxCharWidths())
+                    .imageBounds(stripper.getImageBounds())
+                    .minCharWidth(stripper.getMinCharWidth())
+                    .maxCharWidth(stripper.getMaxCharWidth())
                    .landscape(isLandscape)
                    .rotated(isRotated)
                    .build();

-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements
-                    .getMaxCharWidth(), parsedElements.getMaxCharHeight());
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight());

            Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
                    .getVertical());
@@ -81,8 +81,10 @@ public class PdfSegmentationService {

            page.setPageNumber(pageNumber);
            increaseDocumentStatistics(page, document);
+            page.setImageBounds(parsedElements.getImageBounds());
            pages.add(page);
        }
+
        document.setPages(pages);

        classificationService.classifyDocument(document);
@@ -90,11 +92,9 @@ public class PdfSegmentationService {
        sectionsBuilderService.buildSections(document);

        return document;
-
    }


-
    private void increaseDocumentStatistics(Page page, Document document) {

        if (!page.isLandscape()) {
@@ -10,9 +10,12 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -24,25 +27,46 @@ public class SectionsBuilderService {

        List<AbstractTextContainer> chunkWords = new ArrayList<>();
        List<Paragraph> chunkBlockList = new ArrayList<>();
+        List<Header> headers = new ArrayList<>();
+        List<Footer> footers = new ArrayList<>();
+        List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

        AbstractTextContainer prev = null;

        String lastHeadline = "";
        Table previousTable = null;
        for (Page page : document.getPages()) {
+            List<TextBlock> header = new ArrayList<>();
+            List<TextBlock> footer = new ArrayList<>();
+            List<TextBlock> unclassifiedText = new ArrayList<>();
            for (AbstractTextContainer current : page.getTextBlocks()) {

-                if (current.getClassification() == null || current.getClassification()
-                        .equals("Header") || current.getClassification().equals("Footer")) {
+                if (current.getClassification() == null) {
                    continue;
                }

                current.setPage(page.getPageNumber());

-                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
+                if (current.getClassification().equals("Header")) {
+                    header.add((TextBlock) current);
+                    continue;
+                }
+
+                if (current.getClassification().equals("Footer")) {
+                    footer.add((TextBlock) current);
+                    continue;
+                }
+
+                if (current.getClassification().equals("Other")) {
+                    unclassifiedText.add((TextBlock) current);
+                    continue;
+                }
+
+                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
+                        .startsWith("H ") || !document.isHeadlines()) {
                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                    chunkBlock.setHeadline(lastHeadline);
-                    if(document.isHeadlines()) {
+                    if (document.isHeadlines()) {
                        lastHeadline = current.getText();
                    }
                    chunkBlockList.add(chunkBlock);
@@ -60,6 +84,9 @@ public class SectionsBuilderService {
                chunkWords.add(current);
                prev = current;
            }
+            headers.add(new Header(header));
+            footers.add(new Footer(footer));
+            unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
        }

        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
@@ -67,6 +94,9 @@ public class SectionsBuilderService {
        chunkBlockList.add(chunkBlock);

        document.setParagraphs(chunkBlockList);
+        document.setHeaders(headers);
+        document.setFooters(footers);
+        document.setUnclassifiedTexts(unclassifiedTexts);
    }


@@ -175,9 +205,9 @@ public class SectionsBuilderService {

    private boolean hasInvalidHeaderInformation(Table table) {

-        return table.getRows().stream()
-                .flatMap(row -> row.stream()
-                        .filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
+        return table.getRows()
+                .stream()
+                .flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
                .findAny()
                .isEmpty();

@@ -188,6 +218,9 @@ public class SectionsBuilderService {

        for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
            List<Cell> row = table.getRows().get(i);
+            if (row.size() == 1) {
+                continue;
+            }
            boolean allNonHeader = true;
            for (Cell cell : row) {
                if (cell.isHeaderCell()) {
@@ -110,6 +110,10 @@ public class Table extends AbstractTextContainer {
        // we move from left to right and top to bottom
        for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
            List<Cell> rowCells = rows.get(rowIndex);
+            if(rowCells.size() == 1){
+                continue;
+            }
+
            for (int colIndex = 0; colIndex < rowCells.size(); colIndex++) {
                Cell cell = rowCells.get(colIndex);
                List<Cell> cellsToTheLeft = rowCells.subList(0, colIndex);
@@ -18,9 +18,10 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
@Service
 public class RulingCleaningService {

-    public CleanRulings getCleanRulings(List<Ruling> rulings, float maxCharWidth, float maxCharHeight){
+    public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
+
        if (!rulings.isEmpty()) {
-            snapPoints(rulings, maxCharWidth , maxCharHeight);
+            snapPoints(rulings, minCharWidth, maxCharHeight);
        }

        List<Ruling> vrs = new ArrayList<>();
@@ -39,13 +40,10 @@ public class RulingCleaningService {
        }
        List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);

-        return CleanRulings
-                .builder()
-                .vertical(verticalRulingLines)
-                .horizontal(horizontalRulingLines)
-                .build();
+        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
    }

+
    public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {

        // collect points and keep a Line -> p1,p2 map
@@ -122,12 +120,14 @@ public class RulingCleaningService {


    private List<Ruling> collapseOrientedRulings(List<Ruling> lines) {
+
        int COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT = 1;
        return collapseOrientedRulings(lines, COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT);
    }


    private List<Ruling> collapseOrientedRulings(List<Ruling> lines, int expandAmount) {
+
        ArrayList<Ruling> rv = new ArrayList<>();
        lines.sort((a, b) -> {
            final float diff = a.getPosition() - b.getPosition();
@@ -141,25 +141,24 @@ public class RulingCleaningService {
                final float lastStart = last.getStart();
                final float lastEnd = last.getEnd();

-                final boolean lastFlipped = lastStart            > lastEnd;
+                final boolean lastFlipped = lastStart > lastEnd;
                final boolean nextFlipped = next_line.getStart() > next_line.getEnd();

                boolean differentDirections = nextFlipped != lastFlipped;
-                float nextS = differentDirections ? next_line.getEnd()   : next_line.getStart();
+                float nextS = differentDirections ? next_line.getEnd() : next_line.getStart();
                float nextE = differentDirections ? next_line.getStart() : next_line.getEnd();

                final float newStart = lastFlipped ? Math.max(nextS, lastStart) : Math.min(nextS, lastStart);
-                final float newEnd   = lastFlipped ? Math.min(nextE, lastEnd)   : Math.max(nextE, lastEnd);
+                final float newEnd = lastFlipped ? Math.min(nextE, lastEnd) : Math.max(nextE, lastEnd);
                last.setStartEnd(newStart, newEnd);
                assert !last.oblique();
-            }
-            else if (next_line.length() == 0) {
+            } else if (next_line.length() == 0) {
                continue;
-            }
-            else {
+            } else {
                rv.add(next_line);
            }
        }
        return rv;
    }
+
 }
@@ -51,6 +51,7 @@ import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.model.Comment;
 import com.iqser.red.service.redaction.v1.model.IdRemoval;
+import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.Point;
@@ -81,6 +82,7 @@ public class RedactionIntegrationTest {
    private static final String PUBLISHED_INFORMATION = "published_information";
    private static final String TEST_METHOD = "test_method";
    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";

    private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
    private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
@@ -157,6 +159,7 @@ public class RedactionIntegrationTest {
        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
        when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
+        when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(IMAGE));
        when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
    }

@@ -238,6 +241,11 @@ public class RedactionIntegrationTest {
                        .stream()
                        .map(this::cleanDictionaryEntry)
                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/image.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
    }


@@ -264,6 +272,7 @@ public class RedactionIntegrationTest {
        typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
        typeColorMap.put(FALSE_POSITIVE, "#ffffff");
        typeColorMap.put(PURITY, "#ffe187");
+        typeColorMap.put(IMAGE, "#fcc5fb");

        hintTypeMap.put(VERTEBRATE, true);
        hintTypeMap.put(ADDRESS, false);
@@ -280,6 +289,7 @@ public class RedactionIntegrationTest {
        hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
        hintTypeMap.put(FALSE_POSITIVE, true);
        hintTypeMap.put(PURITY, false);
+        hintTypeMap.put(IMAGE, true);

        caseInSensitiveMap.put(VERTEBRATE, true);
        caseInSensitiveMap.put(ADDRESS, false);
@@ -296,6 +306,7 @@ public class RedactionIntegrationTest {
        caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
        caseInSensitiveMap.put(FALSE_POSITIVE, false);
        caseInSensitiveMap.put(PURITY, false);
+        caseInSensitiveMap.put(IMAGE, true);

        recommendationTypeMap.put(VERTEBRATE, false);
        recommendationTypeMap.put(ADDRESS, false);
@@ -312,6 +323,7 @@ public class RedactionIntegrationTest {
        recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
        recommendationTypeMap.put(FALSE_POSITIVE, false);
        recommendationTypeMap.put(PURITY, false);
+        recommendationTypeMap.put(IMAGE, false);

        rankTypeMap.put(FALSE_POSITIVE, 160);
        rankTypeMap.put(PURITY, 155);
@@ -328,6 +340,7 @@ public class RedactionIntegrationTest {
        rankTypeMap.put(HINT_ONLY, 50);
        rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
        rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
+        rankTypeMap.put(IMAGE, 30);

        colors.setDefaultColor("#acfc00");
        colors.setNotRedacted("#cccccc");
@@ -425,20 +438,28 @@ public class RedactionIntegrationTest {
    @Test
    public void redactionTest() throws IOException {

+//        49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
+//        182 Fludioxonil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
+//        38 A14325E - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf
+//        91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf
+//        95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf
+
        System.out.println("redactionTest");
        long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");

        AnalyzeRequest request = AnalyzeRequest.builder()
                .ruleSetId(TEST_RULESET_ID)
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -463,11 +484,13 @@ public class RedactionIntegrationTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -482,6 +505,8 @@ public class RedactionIntegrationTest {
    @Test
    public void testManualRedaction() throws IOException {

+//        675eba69b0c2917de55462c817adaa05
+
        System.out.println("testManualRedaction");
        long start = System.currentTimeMillis();
        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
@@ -496,8 +521,13 @@ public class RedactionIntegrationTest {
                .text("This is a comment test")
                .build();
        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .id("0836727c3508a0b2ea271da69c04cc2f")
-                .status(Status.REQUESTED)
+                .id("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                .status(Status.DECLINED)
+                .build()));
+        manualRedactions.setForceRedacts(Set.of(ManualForceRedact.builder()
+                .id("675eba69b0c2917de55462c817adaa05")
+                .legalBasis("Something")
+                .status(Status.APPROVED)
                .build()));

        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
@@ -520,11 +550,13 @@ public class RedactionIntegrationTest {
                .manualRedactions(manualRedactions)
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -576,7 +608,7 @@ public class RedactionIntegrationTest {
    public void htmlTablesTest() throws IOException {

        System.out.println("htmlTablesTest");
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");

        RedactionRequest request = RedactionRequest.builder()
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@@ -618,7 +650,6 @@ public class RedactionIntegrationTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

        result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
@@ -640,11 +671,13 @@ public class RedactionIntegrationTest {
                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
                .build();

-
        AnalyzeResult result = redactionController.analyze(request);

-        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
-
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .redactionLog(result.getRedactionLog())
+                .sectionGrid(result.getSectionGrid())
+                .build());

        try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
            fileOutputStream.write(annotateResponse.getDocument());
@@ -1,15 +1,36 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;

 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+
 public class RegExPatternTest {


+    @Test
+    public void testExpand(){
+        String pattern = "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)";
+        String text = ", G.R., Simoneaux,";
+        Pattern compiledPattern = Pattern.compile(pattern, 0);
+        Matcher matcher = compiledPattern.matcher(text);
+
+        while (matcher.find()) {
+            String match = matcher.group(1);
+            if (StringUtils.isNotBlank(match)) {
+                System.out.println(match);
+            }
+        }
+    }
+
+
+
    @Test
    public void testEmailRegEx(){
        String text = "Address: Schwarzwaldalle " +
@@ -7,149 +7,82 @@ global Section section

 // --------------------------------------- CBI rules -------------------------------------------------------------------

+rule "0: Expand CBI Authors with firstname initials"
+    when
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+    then
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+    end
+
+
 rule "1: Redact CBI Authors"
    when
        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
    then
-        section.redact("CBI_author", 1, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redact("CBI_author", 1, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "2: Redacted because Section contains Vertebrate"
+rule "2: Redact CBI Address"
    when
-        Section(matchesType("vertebrate"))
+        Section(matchesType("CBI_address") || matchesType("recommendation_CBI_address"))
    then
-        section.redact("CBI_address", 2, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redact("CBI_address", 2, "Address found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "3: Not Redacted because Section contains no Vertebrate"
+rule "3: Do not redact genitive CBI_author"
    when
-        Section(!matchesType("vertebrate"))
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
    then
-        section.redactNot("CBI_address", 3, "No Vertebrate found");
+        section.expandToHintAnnotationByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, "test_method");
+        section.expandToHintAnnotationByRegEx("recommendation_CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, "test_method");
    end


-rule "4: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
-    then
-        section.redactNot("CBI_address", 4, "Vertebrate and No Redaction Indicator found");
-    end
-
-
-rule "5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
-    then
-        section.redact("CBI_address", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "6: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
+rule "4: Redact Author(s) cells in Tables with Author(s) header"
    when
        Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
    then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "7: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
+rule "5: Redact Author cells in Tables with Author header"
    when
        Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
    then
-        section.redactCell("Author", 7, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "8: Not redacted because Vertebrate Study = N"
+rule "6: Redact and recommand Authors in Tables with Vertebrate study Y/N header"
    when
-        Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
+        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
    then
-        section.redactCell("Author(s)", 8, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redactNot("CBI_address", 8, "Not redacted because row is not a vertebrate study");
-        section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
+        section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "9: Redact if must redact entry is found"
-    when
-        Section(matchesType("must_redact"))
-    then
-        section.redact("CBI_address", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
-    then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 10, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
-    end
-
-
-rule "11: Redact sponsor company"
-    when
-        Section(searchText.toLowerCase().contains("batches produced at"))
-    then
-        section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 11, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("batches produced at", "must_redact");
-    end
-
-
-rule "12: Redact determination of residues"
-    when
-        Section((
-            searchText.toLowerCase.contains("determination of residues") ||
-            searchText.toLowerCase.contains("determination of total residues")
-        ) && (
-            searchText.toLowerCase.contains("livestock") ||
-            searchText.toLowerCase.contains("live stock") ||
-            searchText.toLowerCase.contains("tissue") ||
-            searchText.toLowerCase.contains("tissues") ||
-            searchText.toLowerCase.contains("liver") ||
-            searchText.toLowerCase.contains("muscle") ||
-            searchText.toLowerCase.contains("bovine") ||
-            searchText.toLowerCase.contains("ruminant") ||
-            searchText.toLowerCase.contains("ruminants")
-        ))
-    then
-        section.redact("CBI_address", 12, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("determination of residues", "must_redact");
-        section.addHintAnnotation("determination of total residues", "must_redact");
-        section.addHintAnnotation("livestock", "must_redact");
-        section.addHintAnnotation("live stock", "must_redact");
-        section.addHintAnnotation("tissue", "must_redact");
-        section.addHintAnnotation("tissues", "must_redact");
-        section.addHintAnnotation("liver", "must_redact");
-        section.addHintAnnotation("muscle", "must_redact");
-        section.addHintAnnotation("bovine", "must_redact");
-        section.addHintAnnotation("ruminant", "must_redact");
-        section.addHintAnnotation("ruminants", "must_redact");
-    end
-
-
-rule "13: Redact if CTL/* or BL/*  was found"
+rule "7: Redact if CTL/* or BL/*  was found"
    when
        Section(searchText.contains("CTL/") || searchText.contains("BL/"))
    then
-        section.redact("CBI_address", 13, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("CTL", "must_redact");
-        section.addHintAnnotation("BL", "must_redact");
+        section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
+        section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
    end


-rule "14: Redact and add recommendation for et al. author"
+rule "8: Redact and add recommendation for et al. author"
    when
        Section(searchText.contains("et al"))
    then
-		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
    end


-rule "15: Add recommendation for Addresses in Test Organism sections"
+rule "9: Add recommendation for Addresses in Test Organism sections"
    when
        Section(searchText.contains("Species:") && searchText.contains("Source:"))
    then
@@ -157,7 +90,7 @@ rule "15: Add recommendation for Addresses in Test Organism sections"
    end


-rule "16: Add recommendation for Addresses in Test Animals sections"
+rule "10: Add recommendation for Addresses in Test Animals sections"
    when
        Section(searchText.contains("Species") && searchText.contains("Source"))
    then
@@ -165,34 +98,35 @@ rule "16: Add recommendation for Addresses in Test Animals sections"
    end


-rule "17: Do not redact Names and Addresses if no redaction Indicator is contained"
+rule "11: Do not redact Names and Addresses if Published Information found"
    when
        Section(matchesType("published_information"))
    then
-        section.redactNot("CBI_author", 17, "Vertebrate and Published Information found");
-        section.redactNot("CBI_address", 17, "Vertebrate and Published Information found");
+        section.redactNot("CBI_author", 11, "Published Information found");
+        section.redactNot("CBI_address", 11, "Published Information found");
    end

+
 // --------------------------------------- PII rules -------------------------------------------------------------------


-rule "18: Redacted PII Personal Identification Information"
+rule "12: Redacted PII Personal Identification Information"
    when
        Section(matchesType("PII"))
    then
-        section.redact("PII", 18, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redact("PII", 12, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "19: Redact Emails by RegEx"
+rule "13: Redact Emails by RegEx"
    when
        Section(searchText.contains("@"))
    then
-        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 19, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "20: Redact contact information"
+rule "14: Redact contact information"
    when
       Section(text.contains("Contact point:")
       || text.contains("Phone:")
@@ -208,108 +142,120 @@ rule "20: Redact contact information"
       || text.contains("Telephone No:")
       || text.contains("Fax number:")
       || text.contains("Telephone:")
+       || text.contains("Phone No.")
       || text.contains("European contact:"))
    then
-          section.redactLineAfter("Contact point:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Phone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Email:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("e-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail address:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Alternative contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone No:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("No:", "Fax", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("Contact:", "Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("European contact:",  "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+          section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end


-rule "21: Redact contact information if applicant is found"
+rule "15: Redact contact information if applicant is found"
    when
        Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
    then
-        section.redactLineAfter("Contact point:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Email:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("e-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail address:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Alternative contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone No:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("Contact:", "Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("European contact:",  "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "22: Redact contact information if Producer is found"
+rule "16: Redact contact information if Producer is found"
    when
        Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
    then
-        section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "23: Redact AUTHOR(S)"
+rule "17: Redact AUTHOR(S)"
    when
        Section(searchText.contains("AUTHOR(S):"))
    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 23, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "24: Redact PERFORMING LABORATORY"
+rule "18: Redact PERFORMING LABORATORY"
    when
        Section(searchText.contains("PERFORMING LABORATORY:"))
    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 24, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "25: Redact On behalf of Sequani Ltd.:"
+rule "19: Redact On behalf of Sequani Ltd.:"
    when
        Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
    then
-        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 25, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


-rule "26: Redact On behalf of Syngenta Ltd.:"
+rule "20: Redact On behalf of Syngenta Ltd.:"
    when
        Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
    then
-        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 26, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
+        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end


 // --------------------------------------- other rules -------------------------------------------------------------------

-rule "27: Redact Purity"
+rule "21: Purity Hint"
    when
-        Section(searchText.contains("purity"))
+        Section(searchText.toLowerCase().contains("purity"))
    then
-	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 27, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+rule "22: Redact Must Redact"
+    when
+        Section(matchesType("must_redact"))
+    then
+        section.redact("must_redact", 22, "Must Redact found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
    end
@@ -7,6 +7,15 @@ global Section section

 // --------------------------------------- CBI rules -------------------------------------------------------------------

+rule "0: Expand CBI Authors with firstname initials"
+    when
+        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+    then
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
+    end
+
+
 rule "1: Redacted because Section contains Vertebrate"
    when
        Section(matchesType("vertebrate"))
@@ -10,7 +10,7 @@
        <Root level="warn">
            <AppenderRef ref="CONSOLE" />
        </Root>
-        <Logger name="com.iqser.gin4" level="info" />
+        <Logger name="com.iqser" level="info" />
    </Loggers>

 </Configuration>
Author	SHA1	Message	Date
Dominique Eiflaender	7d507c8fda	Pull request #124 : Adjusted rules to new requirements, add possibility to addRedaction for rules Merge in RED/redaction-service from NewRequirementsRules to master * commit 'b6a13d1ff8a55f1b5ef528af5ea726b5e17b4950': Adjusted rules to new requirements, add possibility to addRedaction for rules	2021-02-18 10:48:26 +01:00
Dominique Eifländer	b6a13d1ff8	Adjusted rules to new requirements, add possibility to addRedaction for rules	2021-02-18 10:33:38 +01:00
Dominique Eiflaender	06e418c34e	Pull request #123 : RED-1070: Fixed not finding annotation on not classified textblocks Merge in RED/redaction-service from RED-1070 to master * commit 'de1dea7ac323dc005baeb0660cc492ab84a88164': RED-1070: Fixed not finding annotation on not classified textblocks	2021-02-18 09:18:59 +01:00
Dominique Eifländer	de1dea7ac3	RED-1070: Fixed not finding annotation on not classified textblocks	2021-02-18 08:58:51 +01:00
Dominique Eiflaender	53509f072e	Pull request #122 : RED-1055: Enabled to force redact ignore annotations Merge in RED/redaction-service from RED-1055 to master * commit 'e0fba8d38ce9858235ca84182f4587d472890ef1': RED-1055: Enabled to force redact ignore annotations	2021-02-12 10:55:04 +01:00
Dominique Eifländer	e0fba8d38c	RED-1055: Enabled to force redact ignore annotations	2021-02-12 10:41:32 +01:00
Lena Maldacker	8989191314	Pull request #121 : Add hint annotation by reg-ex and expand to hint annotation by reg-ex Merge in RED/redaction-service from dismissGen to master * commit '0f263c69b8f2eb7201f062cc263c8d06b7a710a1': Add hint annotation by reg-ex and expand to hint annotation by reg-ex	2021-02-11 17:22:55 +01:00
lmaldacker	0f263c69b8	Add hint annotation by reg-ex and expand to hint annotation by reg-ex	2021-02-11 16:53:07 +01:00
Kresnadi Budisantoso	ea1c2898bc	Remove duplicate build tasks in bamboo specs	2021-02-11 14:11:22 +01:00
Dominique Eiflaender	6d9ed080ce	Pull request #120 : RED-1039: Fixed finding textpositions, RED-1042: Fixed get rectangles per line Merge in RED/redaction-service from RED-1039 to master * commit '00b0cb160342f1857ac0e523f994918057d5fc6b': RED-1039: Fixed finding textpositions, RED-1042: Fixed get rectangles per line	2021-02-08 14:50:02 +01:00
Dominique Eifländer	00b0cb1603	RED-1039: Fixed finding textpositions, RED-1042: Fixed get rectangles per line	2021-02-08 14:09:11 +01:00
Dominique Eiflaender	5ecf21290c	Pull request #119 : RED-1046: Ignore dictionary rank for words that are explicitly set in the rules Merge in RED/redaction-service from RED-1046 to master * commit '8965e7654867ff666c54c841319376c5899e7326': RED-1046: Ignore dictionary rank for words that are explicitly set in the rules	2021-02-05 15:08:15 +01:00
Dominique Eifländer	8965e76548	RED-1046: Ignore dictionary rank for words that are explicitly set in the rules	2021-02-05 14:55:16 +01:00
Dominique Eiflaender	ba8c4dc240	Pull request #118 : RED-1045: Enabled to redact in headers and footers Merge in RED/redaction-service from RED-1045 to master * commit '577db37b11dfd0243d22986a6d224700f0b49693': RED-1045: Enabled to redact in headers and footers	2021-02-05 13:16:55 +01:00
Dominique Eifländer	577db37b11	RED-1045: Enabled to redact in headers and footers	2021-02-05 13:05:12 +01:00
Dominique Eiflaender	154e09b843	Pull request #117 : Fixed several table extraction problems Merge in RED/redaction-service from tableExtractionImprovement to master * commit 'a101b98a400b2a35635f2c5e0e894ac849288c3b': Fixed several table extraction problems	2021-02-04 15:21:12 +01:00
Dominique Eifländer	a101b98a40	Fixed several table extraction problems	2021-02-04 15:08:48 +01:00
Dominique Eiflaender	76ecfdabd3	Pull request #116 : Fixed table extraction problems Merge in RED/redaction-service from TableExtractionProbemFix to master * commit 'fc2ac03691871a8a1f3c8a29756e3a60e63e16bf': Fixed table extraction problems	2021-02-03 14:49:01 +01:00
Dominique Eifländer	fc2ac03691	Fixed table extraction problems	2021-02-03 14:34:29 +01:00
Dominique Eiflaender	7898f6a30f	Pull request #115 : Expand CBI Authors with firstname initials Merge in RED/redaction-service from ExpandAuthors to master * commit '35f3582d0850053c1dc38ef0b149daa47bd9cc56': Expand CBI Authors with firstname initials	2021-02-02 12:35:07 +01:00
Dominique Eifländer	35f3582d08	Expand CBI Authors with firstname initials	2021-02-02 12:20:40 +01:00
Dominique Eiflaender	b4699caaae	Pull request #114 : Fixed position problem for rotated images Merge in RED/redaction-service from rotatedImages to master * commit 'd8e444280b5a025e19785c70c9fcbb11480b5042': Fixed position problem for rotated images	2021-02-01 16:17:19 +01:00
Dominique Eifländer	d8e444280b	Fixed position problem for rotated images	2021-02-01 16:04:45 +01:00
Dominique Eiflaender	ec4db538a7	Pull request #113 : Ignore too small images Merge in RED/redaction-service from IgnoreTooSmall to master * commit '39ca191b9cb5dec5fbb01e8319cbbe361c7cd85e': Ignore too small images	2021-02-01 15:46:57 +01:00
Dominique Eifländer	39ca191b9c	Ignore too small images	2021-02-01 15:14:46 +01:00
Dominique Eiflaender	0cec88f1b4	Pull request #112 : Annotate images Merge in RED/redaction-service from annotateImages to master * commit 'acddfafa5b60d0120a48bdf47fe218bf59e359d2': Annotate images	2021-02-01 14:12:28 +01:00
Dominique Eifländer	acddfafa5b	Annotate images	2021-02-01 13:52:18 +01:00
Dominique Eiflaender	6b68890fcf	Pull request #111 : RED-1019: Fixed returning numberOfPages Merge in RED/redaction-service from RED-1019 to master * commit '1ed99412595b560d92e542d949e52d19d58beb2c': RED-1019: Fixed returning numberOfPages	2021-01-29 13:32:05 +01:00
Dominique Eifländer	1ed9941259	RED-1019: Fixed returning numberOfPages	2021-01-29 13:17:25 +01:00
Timo Bejan	1e7d535a33	Pull request #110 : fixed issue affecting service on restarts Merge in RED/redaction-service from bugfix-annotate to master * commit '1091d6a886243387f23da32d0c678b66b0c21f26': fixed issue affecting service on restarts	2021-01-28 10:17:02 +01:00