Pull request #448 : RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive

Merge in RED/redaction-service from RED-4835 to master * commit '6e46cad2c5466136c7191042e6834eb074fc5911': RED-4835 - cleanup RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive
RED-4835 - cleanup
2022-08-08 16:39:49 +02:00 · 2022-08-08 13:31:22 +03:00 · 2022-08-08 13:30:18 +03:00 · 2022-08-01 16:00:29 +02:00 · 2022-08-01 15:55:22 +02:00 · 2022-08-01 11:04:25 +02:00
168 changed files with 63574 additions and 368 deletions
@@ -26,4 +26,3 @@
 **/.DS_Store
 **/classpath-data.json
 **/dependencies-and-licenses-overview.txt
-/redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/
@@ -32,7 +32,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.17.0</version>
+                <version>1.20.0</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -12,7 +12,7 @@
    <artifactId>redaction-service-api-v1</artifactId>

    <properties>
-        <persistence-service.version>1.240.0</persistence-service.version>
+        <persistence-service.version>1.254.0</persistence-service.version>
    </properties>

    <dependencies>
@@ -35,9 +35,7 @@ public class FloatFrequencyCounter {
    public Float getMostPopular() {
        Map.Entry<Float, Integer> mostPopular = null;
        for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null) {
-                mostPopular = entry;
-            } else if (entry.getValue() >= mostPopular.getValue()) {
+            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
                mostPopular = entry;
            }
        }
@@ -61,9 +59,7 @@ public class FloatFrequencyCounter {
    public Float getHighest() {
        Float highest = null;
        for (Float value : countPerValue.keySet()) {
-            if (highest == null) {
-                highest = value;
-            } else if (value > highest) {
+            if (highest == null || value > highest) {
                highest = value;
            }
        }
@@ -37,9 +37,7 @@ public class StringFrequencyCounter {

        Map.Entry<String, Integer> mostPopular = null;
        for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null) {
-                mostPopular = entry;
-            } else if (entry.getValue() > mostPopular.getValue()) {
+            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
                mostPopular = entry;
            }
        }
@@ -23,16 +23,30 @@ public class TextBlock extends AbstractTextContainer {

    @Builder.Default
    private List<TextPositionSequence> sequences = new ArrayList<>();
+
+    @JsonIgnore
    private int rotation;

+    @JsonIgnore
    private String mostPopularWordFont;
+
+    @JsonIgnore
    private String mostPopularWordStyle;
+
+    @JsonIgnore
    private float mostPopularWordFontSize;
+
+    @JsonIgnore
    private float mostPopularWordHeight;
+
+    @JsonIgnore
    private float mostPopularWordSpaceWidth;

+    @JsonIgnore
    private float highestFontSize;

+
+    @JsonIgnore
    private String classification;


@@ -1,29 +1,33 @@
 package com.iqser.red.service.redaction.v1.server.controller;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.*;
-import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.*;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;

-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.resources.RedactionResource;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

@Slf4j
@RestController
@@ -5,7 +5,6 @@ import org.springframework.beans.BeanUtils;

 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonIgnore;

 import lombok.AllArgsConstructor;
@@ -22,22 +21,20 @@ import lombok.SneakyThrows;
 public class RedTextPosition {

    private String textMatrix;
+    private float[] position;
+
+    @JsonIgnore
    private int rotation;
-    private float y;
+
+    @JsonIgnore
    private float pageHeight;
+
+    @JsonIgnore
    private float pageWidth;
+
    private String unicode;

-    @JsonAlias("xdirAdj")
-    @JsonAttribute(alternativeNames = {"xdirAdj"})
-    private float XDirAdj;
-
-    @JsonAlias("ydirAdj")
-    @JsonAttribute(alternativeNames = {"ydirAdj"})
-    private float YDirAdj;
-    private float width;
-    private float heightDir;
-    private float widthDirAdj;
+    @JsonIgnore
    private float dir;

    // not used in reanalysis
@@ -66,18 +63,37 @@ public class RedTextPosition {

        pos.setTextMatrix(textPosition.getTextMatrix().toString());

+        var position = new float[4];
+
+        position[0] = textPosition.getXDirAdj();
+        position[1] = textPosition.getYDirAdj();
+        position[2] = textPosition.getWidthDirAdj();
+        position[3] = textPosition.getHeightDir();
+
+        pos.setPosition(position);
        return pos;
    }


-    @JsonAlias("xdirAdj")
-    @JsonAttribute(alternativeNames = {"xdirAdj"})
-    public void setXDirAdj(float XDirAdj) {this.XDirAdj = XDirAdj;}
+    @JsonIgnore
+    public float getXDirAdj(){
+        return position[0];
+    }

+    @JsonIgnore
+    public float getYDirAdj(){
+        return position[1];
+    }

-    @JsonAlias("ydirAdj")
-    @JsonAttribute(alternativeNames = {"ydirAdj"})
-    public void setYDirAdj(float YDirAdj) {this.YDirAdj = YDirAdj;}
+    @JsonIgnore
+    public float getWidthDirAdj(){
+        return position[2];
+    }
+
+    @JsonIgnore
+    public float getHeightDir(){
+        return position[3];
+    }


 }
@@ -31,8 +31,10 @@ public class TextPositionSequence implements CharSequence {
    private int page;
    private List<RedTextPosition> textPositions = new ArrayList<>();

-    private float x1;
-    private float x2;
+    private float dir;
+    private int rotation;
+    private float pageHeight;
+    private float pageWidth;


    public TextPositionSequence(int page) {
@@ -41,7 +43,7 @@ public class TextPositionSequence implements CharSequence {
    }


-    public static TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
+    public TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {

        var textPositionSequence = new TextPositionSequence();
        textPositionSequence.textPositions = textPositions;
@@ -55,6 +57,10 @@ public class TextPositionSequence implements CharSequence {

        this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
        this.page = page;
+        this.dir = textPositions.get(0).getDir();
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
    }


@@ -85,7 +91,15 @@ public class TextPositionSequence implements CharSequence {
    @Override
    public TextPositionSequence subSequence(int start, int end) {

-        return fromData(textPositions.subList(start, end), page);
+        var textPositionSequence = new TextPositionSequence();
+        textPositionSequence.textPositions = textPositions.subList(start, end);
+        textPositionSequence.page = page;
+        textPositionSequence.dir = dir;
+        textPositionSequence.rotation = rotation;
+        textPositionSequence.pageHeight = pageHeight;
+        textPositionSequence.pageWidth = pageWidth;
+
+        return textPositionSequence;
    }


@@ -106,15 +120,26 @@ public class TextPositionSequence implements CharSequence {
    }


-    public void add(RedTextPosition textPosition) {
+    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {

        this.textPositions.add(textPosition);
+        this.page = textPositionSequence.getPage();
+        this.dir = textPositionSequence.getDir();
+        this.rotation = textPositionSequence.getRotation();
+        this.pageHeight = textPositionSequence.getPageHeight();
+        this.pageWidth = textPositionSequence.getPageWidth();
    }


    public void add(TextPosition textPosition) {

        this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
+
+        this.dir = textPositions.get(0).getDir();
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
+
    }


@@ -122,7 +147,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getX1() {

-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
            return textPositions.get(0).getYDirAdj() - getTextHeight();
        } else {
            return textPositions.get(0).getXDirAdj();
@@ -134,10 +159,11 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getX2() {

-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
            return textPositions.get(0).getYDirAdj();
        } else {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
+            return textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
        }
    }

@@ -146,15 +172,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getRotationAdjustedY() {

-        return textPositions.get(0).getY();
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getRotationAdjustedX() {
-
-        return textPositions.get(0).getXDirAdj();
+        return textPositions.get(0).getYDirAdj();
    }


@@ -162,10 +180,10 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getY1() {

-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
            return textPositions.get(0).getXDirAdj();
        } else {
-            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj();
+            return pageHeight - textPositions.get(0).getYDirAdj();
        }
    }

@@ -174,10 +192,10 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getY2() {

-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - 2;
        } else {
-            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() + getTextHeight();
+            return pageHeight - textPositions.get(0).getYDirAdj() + getTextHeight();
        }
    }

@@ -249,19 +267,11 @@ public class TextPositionSequence implements CharSequence {
    }


-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public int getRotation() {
-
-        return textPositions.get(0).getRotation();
-    }
-
-
    @JsonIgnore
    @JsonAttribute(ignore = true)
    public Rectangle getRectangle() {

-        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, textPositions.get(0).getRotation(), textPositions.get(0).getDir());
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), rotation, dir);

        float height = getTextHeight();

@@ -270,113 +280,132 @@ public class TextPositionSequence implements CharSequence {
        float posYInit;
        float posYEnd;

-        if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 90f) {
+        if (rotation == 0 && dir == 90f) {

            posYInit = getX1();
-            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
+                    .getWidthDirAdj() - 3;
            posXInit = textPositions.get(0).getYDirAdj() + 2;
            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;

-        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 180f) {
+        } else if (rotation == 0 && dir == 180f) {

-            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXInit = pageWidth - getX1() + 1;
+            posXEnd = pageWidth - getX2() + textPositions.get(0)
+                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;

-        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) {
+        } else if (rotation == 0 && dir == 270f) {

-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
+            posYInit = pageHeight - getX1();
+            posYEnd = pageHeight - getX2() - textPositions.get(0)
+                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXInit = pageWidth - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = pageWidth - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + height;

-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 0.0f) {
+        } else if (rotation == 90 && dir == 0.0f) {

-            posXInit = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
+            posXInit = textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
            posXEnd = textPositions.get(0).getXDirAdj();
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
+            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
+            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + 2;

-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 90.0f) {
+        } else if (rotation == 90 && dir == 90.0f) {
            posXEnd = textPositions.get(0).getYDirAdj() + 2;
            posYInit = getY1();
            posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;

-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 180.0f) {
+        } else if (rotation == 90 && dir == 180.0f) {

-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getXDirAdj() - 4;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getXDirAdj();
-            posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1).getHeightDir();
-            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();
+            posXInit = pageWidth - textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() - 4;
+            posXEnd = pageWidth - textPositions.get(0).getXDirAdj();
+            posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1)
+                    .getHeightDir();
+            posYEnd = textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();

-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 270.0f) {
+        } else if (rotation == 90 && dir == 270.0f) {

-            posXInit = textPositions.get(0).getPageWidth() - getX1();
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posYInit = textPositions.get(0).getPageHeight() - getY1();
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getXDirAdj() - height - 4;
+            posXInit = pageWidth - getX1();
+            posXEnd = pageWidth - textPositions.get(0).getYDirAdj() - 2;
+            posYInit = pageHeight - getY1();
+            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() - height - 4;

-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 0f) {
+        } else if (rotation == 180 && dir == 0f) {

-            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
+            posXEnd = textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
+            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
+            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + 2;

-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 90f) {
+        } else if (rotation == 180 && dir == 90f) {

            posYInit = getX1();
            posYEnd = getX2() - 3;
            posXInit = textPositions.get(0).getYDirAdj() + 2;
            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;

-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
+        } else if (rotation == 180 && dir == 180f) {

-            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXInit = pageWidth - getX1() + 1;
+            posXEnd = pageWidth - getX2() + textPositions.get(0)
+                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;

-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 270.0f) {
+        } else if (rotation == 180 && dir == 270.0f) {

-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
+            posYInit = pageHeight - getX1();
+            posYEnd = pageHeight - getX2() - textPositions.get(0)
+                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
+            posXInit = pageWidth - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = pageWidth - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + height;

-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) {
+        } else if (rotation == 270 && dir == 0.0f) {

-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
+            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
            posYEnd = posYInit + 1;
            posXInit = textPositions.get(0).getXDirAdj();
-            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
+            posXEnd = textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;

-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 90.0f) {
+        } else if (rotation == 270 && dir == 90.0f) {

            posYInit = getX1();
            posYEnd = getX2() - height;
            posXInit = textPositions.get(0).getYDirAdj() + 2;
            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;

-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 180.0f) {
+        } else if (rotation == 270 && dir == 180.0f) {

-            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() - 4;
+            posXInit = pageWidth - getX1() + 1;
+            posXEnd = pageWidth - getX2() - 4;
            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;

-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 270.0f) {
+        } else if (rotation == 270 && dir == 270.0f) {

-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - height;
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
+            posYInit = pageHeight - getX1();
+            posYEnd = pageHeight - getX2() - height;
+            posXInit = pageWidth - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = pageWidth - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + height;

        } else {
            // page rotation = 0 and text direction = 0
-            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
+            posXEnd = textPositions.get(textPositions.size() - 1)
+                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
+            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
+            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() + 2;
        }

        var rectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
@@ -1,19 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import lombok.*;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
@Data
@Builder
@NoArgsConstructor
@@ -24,6 +19,7 @@ public class Entity implements ReasonHolder {
    private String word;
    private String type;
    private boolean redaction;
+    private boolean falsePositive;
    private String redactionReason;
    private String legalBasis;
    private List<EntityPositionSequence> positionSequences = new ArrayList<>();
@@ -100,7 +100,7 @@ public class SearchableText {
                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
                            .charAt(j, caseInsensitive) != ' ') {
-                        partMatch.add(searchSpace.get(i).textPositionAt(j));
+                        partMatch.add(searchSpace.get(i),searchSpace.get(i).textPositionAt(j));
                        if (!(j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
                                .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                            counter++;
@@ -1,23 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.StringUtils;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
@@ -28,10 +10,19 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
-
 import lombok.Builder;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;

@Data
@Slf4j
@@ -78,7 +69,7 @@ public class Section {


    @SuppressWarnings("unused")
-    @WhenCondition
+    @ThenAction
    public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {

        Set<Entity> entitiesOfType = nerEntities.stream()
@@ -87,6 +78,7 @@ public class Section {
        List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
        Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());

        Set<Entity> finalResult = new HashSet<>();

@@ -112,7 +104,7 @@ public class Section {


    @SuppressWarnings("unused")
-    @WhenCondition
+    @ThenAction
    public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType,
                               @Argument(ArgumentType.TYPE) String combineTypes,
                               @Argument(ArgumentType.INTEGER) int maxDistanceBetween,
@@ -886,13 +878,11 @@ public class Section {
            }
        });

-        return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
    }


-
-
-
    private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {

        entities.forEach(entity -> {
@@ -1,18 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
@@ -22,24 +9,21 @@ import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;

@Slf4j
@Service
@@ -51,7 +35,6 @@ public class EntityRedactionService {
    private final SurroundingWordsService surroundingWordsService;


-
    public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer,
                                     AnalyzeRequest analyzeRequest, NerEntities nerEntities) {

@@ -262,7 +245,8 @@ public class EntityRedactionService {
            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
        }

-        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions), nerFound);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
    }


@@ -5,7 +5,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;

 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
@@ -13,7 +12,6 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@@ -114,10 +112,7 @@ public class RedactionLogCreatorService {
                redactionLogEntry.setId(entityPositionSequence.getId());

                if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
-                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
-                            .stream()
-                            .flatMap(seq -> seq.getTextPositions().stream())
-                            .collect(Collectors.toList()), page);
+                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences());

                    redactionLogEntry.getPositions().addAll(rectanglesPerLine);

@@ -134,38 +129,43 @@ public class RedactionLogCreatorService {
    }


-    private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {
+    private List<Rectangle> getRectanglesPerLine(List<TextPositionSequence> textPositionSequences) {

        List<Rectangle> rectangles = new ArrayList<>();
-        if (textPositions.size() == 1) {
-            rectangles.add(TextPositionSequence.fromData(textPositions, page).getRectangle());
-        } else {
-            float x = textPositions.get(0).getXDirAdj();
-            float y = textPositions.get(0).getYDirAdj();
-            float width = textPositions.get(0).getWidthDirAdj();
-            float height = textPositions.get(0).getHeightDir();
-            int startIndex = 0;

-            for (int i = 1; i < textPositions.size(); i++) {
-                float xDirAdj = textPositions.get(i).getXDirAdj();
-                float yDirAdj = textPositions.get(i).getYDirAdj();
-                float widthDir = textPositions.get(i).getWidthDirAdj();
-                float heightDir = textPositions.get(i).getHeightDir();
+        if (textPositionSequences.size() == 1) {
+            rectangles.add(textPositionSequences.get(0).getRectangle());
+            return rectangles;
+        }

-                if (!(isCharInSameLine(y, yDirAdj, height, heightDir) && isCharClose(x, xDirAdj, width))) {
-                    rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, i), page)
-                            .getRectangle());
-                    y = yDirAdj;
-                    width = widthDir;
-                    height = heightDir;
-                    startIndex = i;
-                }
-                x = xDirAdj;
+        TextPositionSequence combinedSequence = new TextPositionSequence();
+        for (int i = 0; i < textPositionSequences.size(); i++) {
+
+            if(combinedSequence.getTextPositions().isEmpty()){
+                combinedSequence = textPositionSequences.get(i);
+                continue;
            }
-            if (startIndex != textPositions.size()) {
-                rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page)
-                        .getRectangle());
+
+            float lastSeqX = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getXDirAdj();
+            float lastSeqY = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getYDirAdj();
+            float lastSeqWidth = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getWidthDirAdj();
+            float lastSeqHeight = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getHeightDir();
+
+            float currentSeqX = textPositionSequences.get(i).getTextPositions().get(0).getXDirAdj();
+            float currentSeqY = textPositionSequences.get(i).getTextPositions().get(0).getYDirAdj();
+            float currentSeqHeight = textPositionSequences.get(i).getTextPositions().get(0).getHeightDir();
+
+            if (isCharInSameLine(lastSeqY, currentSeqY, lastSeqHeight, currentSeqHeight) && isCharClose(lastSeqX, currentSeqX, lastSeqWidth)) {
+                combinedSequence.getTextPositions().addAll(textPositionSequences.get(i).getTextPositions());
+            } else {
+                rectangles.add(combinedSequence.getRectangle());
+                combinedSequence = textPositionSequences.get(i);
            }
+
+        }
+
+        if(!combinedSequence.getTextPositions().isEmpty()) {
+            rectangles.add(combinedSequence.getRectangle());
        }

        return rectangles;
@@ -207,7 +207,8 @@ public class RedactionLogCreatorService {
                .redacted(entity.isRedaction())
                .isHint(isHint(entity.getType(), dossierTemplateId))
                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
+                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType()
+                        .equals(EntityType.FALSE_RECOMMENDATION))
                .section(entity.getHeadline())
                .sectionNumber(entity.getSectionNumber())
                .matchedRule(entity.getMatchedRule())
@@ -1,28 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

+import java.util.*;
+import java.util.stream.Collectors;
+
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
@@ -39,12 +26,10 @@ public class EntitySearchUtils {

        if (details.getEntityType() == EntityType.RECOMMENDATION) {
            Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
-            removeFalsePositives(found, falseRecommendations);
-            found.addAll(falseRecommendations);
+            markFalsePositives(found, falseRecommendations);
        } else {
            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
-            removeFalsePositives(found, falsePositives);
-            found.addAll(falsePositives);
+            markFalsePositives(found, falsePositives);
        }

        return found;
@@ -93,7 +78,7 @@ public class EntitySearchUtils {
            }
        }

-        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()){
+        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()) {
            applyResizeRedactions(entities, manualRedactions);
        }

@@ -131,7 +116,11 @@ public class EntitySearchUtils {
    }


-    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
+    public void markFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
+
+        if (entities.size() == 2) {
+            log.info("asd");
+        }

        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : falsePositives) {
@@ -142,8 +131,12 @@ public class EntitySearchUtils {
                }
            }
        }
-        wordsToRemove.forEach(entities::remove);
-        entities.removeAll(falsePositives);
+        wordsToRemove.forEach(e -> e.setFalsePositive(true));
+        for (var entity : entities) {
+            if (falsePositives.contains(entity)) {
+                entity.setFalsePositive(true);
+            }
+        }
    }


@@ -152,13 +145,22 @@ public class EntitySearchUtils {
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
            for (Entity inner : entities) {
+//                // skip cross-type false positives
+//                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
+//                    continue;
+//                }
+//                // skip cross-type false reccommendations
+//                if(word.getEntityType() == EntityType.FALSE_RECOMMENDATION && inner.getEntityType() == EntityType.RECOMMENDATION && !inner.getType().equals(word.getType())){
+//                    continue;
+//                }
+
                if (inner.getWord().length() < word.getWord()
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                        wordsToRemove.add(word);
-                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
                            inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
-                        if(inner.isResized()){
+                        if (inner.isResized()) {
                            wordsToRemove.add(word);
                        } else {
                            wordsToRemove.add(inner);
@@ -1,8 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.storage;

-import com.dslplatform.json.DslJson;
-import com.dslplatform.json.runtime.Settings;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.InputStream;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
@@ -12,16 +13,12 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
+
 import io.micrometer.core.annotation.Timed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.stereotype.Service;
-
-import java.io.IOException;
-import java.io.InputStream;

@Slf4j
@Service
@@ -13,13 +13,20 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
 public abstract class AbstractTextContainer {

+    @JsonIgnore
    protected float minX;
+    @JsonIgnore
    protected float maxX;
+    @JsonIgnore
    protected float minY;
+    @JsonIgnore
    protected float maxY;
+    @JsonIgnore
    protected String classification;
+    @JsonIgnore
    protected int page;

+    @JsonIgnore
    private Orientation orientation = Orientation.NONE;

    public abstract String getText();
@@ -1,14 +1,5 @@
 package com.iqser.red.service.redaction.v1.server;

-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-import lombok.SneakyThrows;
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -19,6 +10,16 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

+import org.apache.commons.io.IOUtils;
+import org.springframework.core.io.InputStreamResource;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
 public class FileSystemBackedStorageService implements StorageService {

    private final Map<String, File> dataMap = new HashMap<>();
@@ -62,11 +63,8 @@ public class FileSystemBackedStorageService implements StorageService {
    }

    private ObjectMapper getMapper() {
-        var objectMapper = new ObjectMapper();
-        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        objectMapper.registerModule(new JavaTimeModule());
-        objectMapper.findAndRegisterModules();
-        return objectMapper;
+
+        return ObjectMapperFactory.create();
    }

    @Override
@@ -27,7 +27,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
@@ -114,10 +113,13 @@ public class RulesTest {
    private static final String IMPORTED_REDACTION = "imported_redaction";
    private static final String PII = "PII";
    private static final String RESOURCES_PATH = "src/test/resources/";
-    private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
+    private static final String REDACTION_LOG_PATH = "RedactionLog/";
+    private static final String REDACTION_LOG_RESOURCES_PATH = RESOURCES_PATH + REDACTION_LOG_PATH;
    private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
    private final static String TEST_DOSSIER_ID = "123";
    private String TEST_FILE_ID = "123";
+    private int counter = 0;
+    private int fileSize = 0;
    private final Map<String, List<String>> dictionary = new HashMap<>();
    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
    private final Map<String, List<String>> falsePositive = new HashMap<>();
@@ -176,7 +178,9 @@ public class RulesTest {
                .hexColor("#ffe187")
                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .build()));

        mockDictionaryCalls(null);
        mockDictionaryCalls(0L);
@@ -196,7 +200,7 @@ public class RulesTest {

    /**
     * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
-     * If the RedactionLog already exists, the generating will be skipped
+     * If the RedactionLog already exists, it will be overwritten
     * Test is ignored, because it's for manual tests.
     */
    @Ignore
@@ -204,20 +208,22 @@ public class RulesTest {
    public void generateRedactionLogForOneFile() {

        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
-        generateAndSaveRedactionLog(fileName);
+        fileSize = 1;
+        generateAndSaveRedactionLog(fileName, true);
    }


    /**
     * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
-     * If a RedactionLog already exists, the generating will be skipped
+     * If a RedactionLog already exists, the generating for this will be skipped
+     * Please commit generated files, if not the following tests will fail
     */
-    @IfProfileValue(name = "test-groups", value = "rules-test")
    @Test
    public void generateRedactionLogForAllFiles() {

        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
        System.out.println("Will generate RedactionLog for " + files.size() + " files.");
+        fileSize = files.size();
        files.forEach(this::generateAndSaveRedactionLog);
    }

@@ -225,20 +231,19 @@ public class RulesTest {
    /**
     * Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
     * If RedactionLog Json does not exist, test will fail.
-     * Test is ignored, because it's for manual tests.
     */
-    @Ignore
    @Test
    public void analyseFileAndCompareRedactionLog() {

-        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
+        String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf";
+        fileSize = 1;
        analyseFileAndCompareRedactionLog(fileName);
    }


    /**
     * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
-     * If RedactionLogs Json does not exist, test will fail.
+     * If RedactionLog Json for one file does not exist, whole test will fail.
     */
    @IfProfileValue(name = "test-groups", value = "rules-test")
    @Test
@@ -246,20 +251,28 @@ public class RulesTest {

        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
        System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
+        fileSize = files.size();
        files.forEach(this::analyseFileAndCompareRedactionLog);
    }


-    @SneakyThrows
    private void generateAndSaveRedactionLog(String fileName) {

-        if (doesRedactionLogExist(fileName)) {
-            System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
+        generateAndSaveRedactionLog(fileName, false);
+    }
+
+
+    @SneakyThrows
+    private void generateAndSaveRedactionLog(String fileName, boolean force) {
+
+        counter++;
+        if (!force && doesRedactionLogExist(fileName)) {
+            System.out.println("(" + counter + "/" + fileSize + ") Skip generating RedactionLog as Json for " + fileName + " because it already exists.");

        } else {
            generateTestFileId(fileName);

-            System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+            System.out.println("(" + counter + "/" + fileSize + ") Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);

            loadNerForTest();

@@ -278,8 +291,9 @@ public class RulesTest {
    @SneakyThrows
    public void analyseFileAndCompareRedactionLog(String fileName) {

+        counter++;
        generateTestFileId(fileName);
-        System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
+        System.out.println("(" + counter + "/" + fileSize + ") Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");

        RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);

@@ -299,7 +313,7 @@ public class RulesTest {
        assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
        assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());

-        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
+        assertThat(redactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size());
        assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());

        for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
@@ -387,7 +401,7 @@ public class RulesTest {
                assertThat(savedManualChange).isPresent();
            }

-            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
+            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(new Engine[0]));

            assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
            assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
@@ -399,7 +413,7 @@ public class RulesTest {
    private boolean doesRedactionLogExist(String pdfFileName) {

        File pdfFile = new File(pdfFileName);
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
        File file = new File(directory, fileName);
        return file.exists();
@@ -421,7 +435,7 @@ public class RulesTest {

        File pdfFile = new File(pdfFileName);

-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
        File dr = new File(directory);
        boolean created = dr.mkdirs();
        if (created) {
@@ -441,8 +455,20 @@ public class RulesTest {
    private RedactionLog loadSavedRedactionLog(String pdfFileName) {

        File pdfFile = new File(pdfFileName);
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = cleanPath(pdfFile.getParentFile().getPath());
+        if (StringUtils.contains(directory, RESOURCES_PATH)) {
+            if (!StringUtils.endsWith(directory, "/")) {
+                directory = directory + "/";
+            }
+            directory = directory + REDACTION_LOG_PATH;
+        } else {
+            if (StringUtils.startsWith(directory, "/")) {
+                directory = StringUtils.substring(directory, 1);
+            }
+            directory = REDACTION_LOG_RESOURCES_PATH + directory;
+        }
        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+
        File file = new File(directory, fileName);
        assertThat(file).exists();

@@ -450,6 +476,31 @@ public class RulesTest {
    }


+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = cleanPath(path.toAbsolutePath().toString());
+                    int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length();
+
+                    fileNames.add(StringUtils.substring(absolutePath, pos));
+                }
+            }
+        }
+        return fileNames;
+    }
+
+
+    private String cleanPath(String path) {
+
+        return StringUtils.replace(path, "\\", "/");
+    }
+
+
    @SneakyThrows
    private void loadNerForTest() {

@@ -588,24 +639,6 @@ public class RulesTest {
    }


-    @SneakyThrows
-    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
-
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
-            for (Path path : stream) {
-                if (path.toFile().isDirectory()) {
-                    getFileNames(fileNames, path);
-                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
-                    String absolutePath = path.toAbsolutePath().toString();
-                    int pos = StringUtils.indexOf(absolutePath, StringUtils.replace(RESOURCES_PATH, "/", "\\")) + 18;
-                    fileNames.add(StringUtils.substring(absolutePath, pos));
-                }
-            }
-        }
-        return fileNames;
-    }
-
-
    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {

        return Type.builder()
@@ -1,14 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.annotate;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.*;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import lombok.RequiredArgsConstructor;
+import java.awt.Color;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -22,12 +23,21 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;

-import java.awt.*;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.CellRectangle;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.SectionRectangle;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;

@Service
@RequiredArgsConstructor
@@ -5,9 +5,11 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

 import java.time.OffsetDateTime;
@@ -19,6 +21,9 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest

    public static final String FILE_NAME = "test-file";

+    @Autowired
+    private AnnotationService annotationService;
+
    @Test
    @SneakyThrows
    public void testFile() {
@@ -81,7 +86,19 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
        redactionMessageReceiver.receiveAnalyzeRequest(in, false);


-        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        var redactionLog = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        redactionLog.getRedactionLogEntry().forEach(entry ->{
+//            log.info("Entry {}", entry);
+//        });
+//
+//        var annotated = annotationService.annotate(AnnotateRequest.builder().fileId("fileId").dossierId("dossierId").build());
+//        annotated.getDocument();
+//
+//        File tempFile = File.createTempFile("annotated",".pdf");
+//        IOUtils.write(annotated.getDocument(), new FileOutputStream(tempFile));
+//        log.warn("File saved to: {}",tempFile.getAbsolutePath());
+//        Runtime.getRuntime().exec("open "+tempFile.getAbsolutePath());
    }

+
 }
@@ -1,17 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.realdata;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.nullable;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.*;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -30,27 +34,14 @@ import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.test.context.junit4.SpringRunner;

-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
-import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;

-import lombok.SneakyThrows;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -143,6 +134,7 @@ public class LiveDataIntegrationTest {
            if (found.isPresent()) {
                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
                type.setEntries(getEntries(typeName, type.getTypeId()));
+                type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId()));

                return type;
            } else {
@@ -196,7 +188,22 @@ public class LiveDataIntegrationTest {

        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
        for (var resource : dictionaryResources) {
-            if (Objects.requireNonNull(resource.getFilename()).contains(typeName)) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) {
+
+                List<String> lines = IOUtils.readLines(resource.getInputStream());
+                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
+
+            }
+        }
+        return new ArrayList<>();
+    }
+
+    @SneakyThrows
+    private List<DictionaryEntry> getFalsePositiveEntries(String typeName, String typeId) {
+
+        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
+        for (var resource : dictionaryResources) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) {

                List<String> lines = IOUtils.readLines(resource.getInputStream());
                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
@@ -1,15 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import org.apache.commons.lang3.StringUtils;
-import org.junit.Test;
-
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;

 public class RegExPatternTest {

@@ -0,0 +1,533 @@
+{
+  "analysisVersion": 1,
+  "analysisNumber": 0,
+  "redactionLogEntry": [
+    {
+      "id": "96c1362cb4ea87461d8f5f141df1912a",
+      "type": "PII",
+      "value": "patrick.gardinal@syngenta.com",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 368.44
+          },
+          "width": 141.39014,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 10,
+      "textBefore": "60 51 E-mail: ",
+      "textAfter": null,
+      "comments": [],
+      "startOffset": 295,
+      "endOffset": 324,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "74b528293158d5266ace1eaf9ee2ad80",
+      "type": "PII",
+      "value": "+41 (0) 61 323 60 51",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "Producer of the active substance: Cyprodinil",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 169.96002
+          },
+          "width": 94.37213,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 12,
+      "textBefore": "Gardinal Telephone number: ",
+      "textAfter": " E-mail: patrick.gardinal@syngenta.com",
+      "comments": [],
+      "startOffset": 188,
+      "endOffset": 208,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "d282cd8f02e2628c14342ed52d69c517",
+      "type": "PII",
+      "value": "+41 (61) 323 6152",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.1 Applicant",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 610.12
+          },
+          "width": 83.43994,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 9,
+      "textBefore": "Micheletti Telephone number: ",
+      "textAfter": " E-mail: sebastien.micheletti@syngenta.com",
+      "comments": [],
+      "startOffset": 164,
+      "endOffset": 181,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "245cfd060d4fb078bb211fabfce6d470",
+      "type": "PII",
+      "value": "patrick.gardinal@syngenta.com",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "Producer of the active substance: Cyprodinil",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 144.64001
+          },
+          "width": 141.39014,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 12,
+      "textBefore": "60 51 E-mail: ",
+      "textAfter": null,
+      "comments": [],
+      "startOffset": 217,
+      "endOffset": 246,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "67dc43606b8e1489a0594e5b779fa428",
+      "type": "PII",
+      "value": "Patrick Gardinal",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 419.08
+          },
+          "width": 73.440186,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 10,
+      "textBefore": "Basel; Switzerland Contact: ",
+      "textAfter": " Telephone number:",
+      "comments": [],
+      "startOffset": 231,
+      "endOffset": 247,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "de79428457282a0781a827b322b5732a",
+      "type": "PII",
+      "value": "sebastien.micheletti@syngenta.com",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.1 Applicant",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 584.8
+          },
+          "width": 161.04922,
+          "height": 10.98,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 9,
+      "textBefore": "323 6152 E-mail: ",
+      "textAfter": null,
+      "comments": [],
+      "startOffset": 190,
+      "endOffset": 223,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "41340ab190d66ca6dce0ba965ce95f59",
+      "type": "PII",
+      "value": "Patrick Gardinal",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "Producer of the active substance: Cyprodinil",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 195.28003
+          },
+          "width": 73.440186,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 12,
+      "textBefore": "Basel; Switzerland Contact: ",
+      "textAfter": " Telephone number:",
+      "comments": [],
+      "startOffset": 153,
+      "endOffset": 169,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "02dd2efb100839e705cdcf6f3141d9d5",
+      "type": "PII",
+      "value": "Sebastien Micheletti",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.1 Applicant",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 634.84
+          },
+          "width": 91.21478,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 9,
+      "textBefore": "Basel; Switzerland Contact: ",
+      "textAfter": " Telephone number:",
+      "comments": [],
+      "startOffset": 125,
+      "endOffset": 145,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    },
+    {
+      "id": "530c4f47f0afc48432bd849a90918f41",
+      "type": "PII",
+      "value": "+41 (0) 61 323 60 51",
+      "reason": "Personal information found",
+      "matchedRule": 23,
+      "rectangle": false,
+      "legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
+      "imported": false,
+      "redacted": true,
+      "section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
+      "color": [
+        0.4,
+        0.8,
+        1.0
+      ],
+      "positions": [
+        {
+          "topLeft": {
+            "x": 286.92,
+            "y": 393.76
+          },
+          "width": 94.37213,
+          "height": 11.52,
+          "page": 5
+        }
+      ],
+      "sectionNumber": 10,
+      "textBefore": "Gardinal Telephone number: ",
+      "textAfter": " E-mail: patrick.gardinal@syngenta.com",
+      "comments": [],
+      "startOffset": 266,
+      "endOffset": 286,
+      "imageHasTransparency": false,
+      "excluded": false,
+      "sourceId": null,
+      "changes": [
+        {
+          "analysisNumber": 0,
+          "type": "ADDED",
+          "dateTime": "2022-07-25T11:12:04.3835183+02:00"
+        }
+      ],
+      "manualChanges": [],
+      "engines": [
+        "RULE"
+      ],
+      "reference": [],
+      "importedRedactionIntersections": [],
+      "dossierDictionaryEntry": false,
+      "localManualRedaction": false,
+      "dictionaryEntry": false,
+      "image": false,
+      "falsePositive": false,
+      "manuallyRemoved": false,
+      "hint": false,
+      "recommendation": false
+    }
+  ],
+  "legalBasis": [],
+  "dictionaryVersion": 0,
+  "dossierDictionaryVersion": 0,
+  "rulesVersion": 0,
+  "legalBasisVersion": 0
+}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"12ac3d26ce82a77e469c64d56081662d","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Table in: COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.8,"y":639.64},"width":55.039078,"height":11.52,"page":4}],"sectionNumber":3,"textBefore":"Report: Document F/02, ","textAfter":". (2014). Cyprodinil","comments":[],"startOffset":23,"endOffset":35,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dd614353319e7c5a8cf8b3346dc12fac","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":306.84,"y":667.12},"width":60.71997,"height":12.0,"page":4}],"sectionNumber":5,"textBefore":"F report (","textAfter":"., 2014).","comments":[],"startOffset":247,"endOffset":259,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"36a5e8909a18a5080dd423afb8efd7a7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":159.0,"y":320.0},"width":10.208038,"height":-21.999977,"page":1}],"sectionNumber":3,"textBefore":"Product Metabolism and ","textAfter":" 1 of","comments":[],"startOffset":368,"endOffset":372,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dca119c3aadd348bc43c0b700c0075c3","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.0,"y":81.0},"width":10.208031,"height":-22.0,"page":1}],"sectionNumber":3,"textBefore":"Pages RAM 465/02 ","textAfter":" 1 Tel:","comments":[],"startOffset":399,"endOffset":403,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"ef6e0f0ed413dbd7299d992d9a50840b","type":"CBI_author","value":"Crook","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":374.0,"y":389.0},"width":10.515198,"height":-27.02501,"page":1}],"sectionNumber":5,"textBefore":"Author : SJ ","textAfter":" Analytical Science","comments":[],"startOffset":12,"endOffset":17,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"0fac36714dd9d1c745565c6dfe06b9b7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":514.14215,"y":47.94165},"width":24.39972,"height":11.591161,"page":2}],"sectionNumber":4,"textBefore":"RAM 465/02 ","textAfter":" 2 Summary","comments":[],"startOffset":11,"endOffset":15,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"71770acab1fa63f46c026966890b5231","type":"vertebrate","value":"rabbit","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[1.0,0.52156866,0.96862745],"positions":[{"topLeft":{"x":77.73999,"y":537.62},"width":-11.043678,"height":36.630165,"page":1}],"sectionNumber":4,"textBefore":null,"textAfter":null,"comments":[],"startOffset":41,"endOffset":47,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398329.875800000}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":true,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"64eb5ff4ea1db8665fb6e57c5369b411","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Text in table","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":154.48782,"y":654.18},"width":9.992157,"height":19.882175,"page":1}],"sectionNumber":8,"textBefore":null,"textAfter":" 1 of","comments":[],"startOffset":54,"endOffset":58,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398731.072100800}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"9f6837120752b85c20e27d4cb199bd51","type":"CBI_author","value":"B. Foo","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":92.520004,"y":712.96},"width":31.589584,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":null,"textAfter":" F. Bar","comments":[],"startOffset":0,"endOffset":6,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"fb930b7f3acf65a74b0b59fee32cc6fa","type":"CBI_author","value":"F. Bar","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":125.92424,"y":712.96},"width":29.137009,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":"B. Foo ","textAfter":" Foo, B.","comments":[],"startOffset":7,"endOffset":13,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"435224747b87ed87b60ab363e6610fba","type":"CBI_author","value":"Foo","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":92.520004,"y":686.8},"width":18.338272,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":"Foo F. Bar ","textAfter":", B.","comments":[],"startOffset":14,"endOffset":17,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Timo Bejan	3013dc95f6	Pull request #448 : RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive Merge in RED/redaction-service from RED-4835 to master * commit '6e46cad2c5466136c7191042e6834eb074fc5911': RED-4835 - cleanup RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive	2022-08-08 16:39:49 +02:00
Timo Bejan	6e46cad2c5	RED-4835 - cleanup	2022-08-08 13:31:22 +03:00
Timo Bejan	c41e230f85	RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive	2022-08-08 13:30:18 +03:00
Ali Oezyetimoglu	16ea8364df	Pull request #447 : RED-4867: Sonar issues: RedactionService Merge in RED/redaction-service from RED-4867-rs1 to master * commit '3ec1e20e417e65907448930cf7f8eb9713874fb9': RED-4867: Sonar issues: RedactionService	2022-08-01 16:00:29 +02:00
Ali Oezyetimoglu	3ec1e20e41	RED-4867: Sonar issues: RedactionService	2022-08-01 15:55:22 +02:00
Dominique Eiflaender	ce1f8e9117	Pull request #434 : RED-4753: Reduced size of TEXT File Merge in RED/redaction-service from RED-4753 to master * commit '55bebe8541cfded8a456b63dbd8d8b7e79bab33f': RED-4753: Reduced size of TEXT File a bit more RED-4753: Reduced size of TEXT File	2022-08-01 11:04:25 +02:00
Kresnadi Budisantoso	b7e97890be	Pull request #441 : RED-4829 Fix wrong annotation Merge in RED/redaction-service from kbudisantoso/Sectionjava-1659012500262 to master * commit '47de526b4bd451d302317647cb05ac685e069622': RED-4829 Fix wrong annotation	2022-08-01 09:07:11 +02:00
Dominique Eiflaender	ec49945682	Pull request #446 : RED-4843: Disable jsondsl per default Merge in RED/redaction-service from RED-4843 to master * commit 'b64222938c64eeb006d62807a705918ffd2c585b': RED-4843: Disable jsondsl per default	2022-07-29 12:47:38 +02:00
deiflaender	b64222938c	RED-4843: Disable jsondsl per default	2022-07-29 12:43:35 +02:00
Timo Bejan	e4823e20d4	Pull request #444 : RED-4799 - remove FP entities from redactionlog Merge in RED/redaction-service from RED-4799-3 to master * commit '53428ec65296b9fac2c206e7bd89916c933fb5c8': RED-4799 - remove FP entities from redactionlog	2022-07-29 10:53:51 +02:00
Timo Bejan	53428ec652	RED-4799 - remove FP entities from redactionlog	2022-07-29 11:50:38 +03:00
Dominique Eiflaender	ca270fb7de	Pull request #443 : RED-4843: Upgraded storage commons Merge in RED/redaction-service from RED-4843 to master * commit '06fe0dbd97cdf2011be17df07f8c409a221530a4': RED-4843: Upgraded storage commons	2022-07-29 10:41:24 +02:00
deiflaender	06fe0dbd97	RED-4843: Upgraded storage commons	2022-07-29 10:27:18 +02:00
Kresnadi Budisantoso	47de526b4b	RED-4829 Fix wrong annotation	2022-07-28 14:48:39 +02:00
Timo Bejan	1d8e86e4f6	Pull request #439 : RED-4799 Merge in RED/redaction-service from RED-4799-fix to master * commit '2b7972315f655bd296a03cae8cc5e4846d47bbce': RED-4799	2022-07-27 11:58:11 +02:00
Timo Bejan	2b7972315f	RED-4799	2022-07-27 12:54:51 +03:00
Philipp Schramm	0af853dbfc	Pull request #436 : RED-4510: Added generated RedactionLog Json files and optimizations for RulesTest.java for Bamboo Merge in RED/redaction-service from RED-4510-Rework to master * commit '0544a42117a742de70325cd1a84e57d6829aaa25': RED-4510: Added generated RedactionLog Json files and optimizations for RulesTest.java for Bamboo	2022-07-26 17:08:59 +02:00
Philipp Schramm	0544a42117	RED-4510: Added generated RedactionLog Json files and optimizations for RulesTest.java for Bamboo	2022-07-26 17:06:04 +02:00
deiflaender	55bebe8541	RED-4753: Reduced size of TEXT File a bit more	2022-07-26 11:55:25 +02:00
Timo Bejan	8bdfd6747d	Pull request #435 : RED-4686 Merge in RED/redaction-service from tbejan/pomxml-1658825710723 to master * commit '80c0c25c069995790d6bd043657f50dc8de0a451': RED-4686	2022-07-26 11:01:22 +02:00
Timo Bejan	80c0c25c06	RED-4686	2022-07-26 10:55:17 +02:00
deiflaender	1cc66a3092	RED-4753: Reduced size of TEXT File	2022-07-26 09:45:46 +02:00
Timo Bejan	17baf9c0eb	Pull request #433 : RED-4686 - updated cyclic deps Merge in RED/redaction-service from RED-4686 to master * commit '090a6196000dfcd7a447c54224fba8eda4605380': RED-4686 - updated cyclic deps	2022-07-26 09:25:58 +02:00
Timo Bejan	090a619600	RED-4686 - updated cyclic deps	2022-07-26 10:22:50 +03:00
				`@@ -0,0 +1 @@`
				{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"12ac3d26ce82a77e469c64d56081662d","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Table in: COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.8,"y":639.64},"width":55.039078,"height":11.52,"page":4}],"sectionNumber":3,"textBefore":"Report: Document F/02, ","textAfter":". (2014). Cyprodinil","comments":[],"startOffset":23,"endOffset":35,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dd614353319e7c5a8cf8b3346dc12fac","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":306.84,"y":667.12},"width":60.71997,"height":12.0,"page":4}],"sectionNumber":5,"textBefore":"F report (","textAfter":"., 2014).","comments":[],"startOffset":247,"endOffset":259,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
				`@@ -0,0 +1 @@`
				`{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}`