Pull request #464 : RSS-86: Added new rule function redactLineAfterAcrossColumns

Merge in RED/redaction-service from RSS-86 to master * commit 'a5f27cfa4cf81bfe1cb29016d18d3fc40e77624d': RSS-86: Added new rule function redactLineAfterAcrossColumns
RSS-86: Added new rule function redactLineAfterAcrossColumns
2022-09-08 13:47:45 +02:00 · 2022-09-08 13:05:13 +02:00 · 2022-09-08 12:17:20 +02:00 · 2022-09-08 12:10:07 +02:00 · 2022-09-07 14:37:26 +02:00 · 2022-09-07 14:31:08 +02:00
215 changed files with 1182 additions and 597 deletions
@@ -26,4 +26,3 @@
 **/.DS_Store
 **/classpath-data.json
 **/dependencies-and-licenses-overview.txt
-/redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/
@@ -1,4 +1,4 @@
-FROM red/redaction-service-base-v1:1.0.0
+FROM red/redaction-service-base-v1:2.0.0

 ARG PLATFORM_JAR

@@ -32,7 +32,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.17.0</version>
+                <version>1.20.0</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -12,7 +12,7 @@
    <artifactId>redaction-service-api-v1</artifactId>

    <properties>
-        <persistence-service.version>1.240.0</persistence-service.version>
+        <persistence-service.version>1.254.0</persistence-service.version>
    </properties>

    <dependencies>
@@ -35,9 +35,7 @@ public class FloatFrequencyCounter {
    public Float getMostPopular() {
        Map.Entry<Float, Integer> mostPopular = null;
        for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null) {
-                mostPopular = entry;
-            } else if (entry.getValue() >= mostPopular.getValue()) {
+            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
                mostPopular = entry;
            }
        }
@@ -61,9 +59,7 @@ public class FloatFrequencyCounter {
    public Float getHighest() {
        Float highest = null;
        for (Float value : countPerValue.keySet()) {
-            if (highest == null) {
-                highest = value;
-            } else if (value > highest) {
+            if (highest == null || value > highest) {
                highest = value;
            }
        }
@@ -27,11 +27,15 @@ public class SectionText {
    private boolean isTable;
    private String headline;

+    @Builder.Default
    private List<SectionArea> sectionAreas = new ArrayList<>();
+    @Builder.Default
    private Set<Image> images = new HashSet<>();
-
+    @Builder.Default
    private List<TextBlock> textBlocks = new ArrayList<>();
+    @Builder.Default
    private Map<String, CellValue> tabularData = new HashMap<>();
+    @Builder.Default
    private List<Integer> cellStarts = new ArrayList<>();


@@ -37,9 +37,7 @@ public class StringFrequencyCounter {

        Map.Entry<String, Integer> mostPopular = null;
        for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null) {
-                mostPopular = entry;
-            } else if (entry.getValue() > mostPopular.getValue()) {
+            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
                mostPopular = entry;
            }
        }
@@ -23,16 +23,30 @@ public class TextBlock extends AbstractTextContainer {

    @Builder.Default
    private List<TextPositionSequence> sequences = new ArrayList<>();
+
+    @JsonIgnore
    private int rotation;

+    @JsonIgnore
    private String mostPopularWordFont;
+
+    @JsonIgnore
    private String mostPopularWordStyle;
+
+    @JsonIgnore
    private float mostPopularWordFontSize;
+
+    @JsonIgnore
    private float mostPopularWordHeight;
+
+    @JsonIgnore
    private float mostPopularWordSpaceWidth;

+    @JsonIgnore
    private float highestFontSize;

+
+    @JsonIgnore
    private String classification;


@@ -1,29 +1,34 @@
 package com.iqser.red.service.redaction.v1.server.controller;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.*;
-import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.*;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;

-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.resources.RedactionResource;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

@Slf4j
@RestController
@@ -37,6 +42,7 @@ public class RedactionController implements RedactionResource {
    private final RedactionLogMergeService redactionLogMergeService;
    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;

+
    @Override
    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {

@@ -118,14 +124,17 @@ public class RedactionController implements RedactionResource {
    @Override
    public void testRules(@RequestBody String rules) {

-        droolsExecutionService.testRules(rules);
+        try {
+            droolsExecutionService.testRules(rules);
+        } catch (Exception e) {
+            throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
+        }
    }


    @Override
    public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {

-
        return redactionLogMergeService.provideRedactionLog(redactionRequest);
    }

@@ -134,19 +143,14 @@ public class RedactionController implements RedactionResource {

        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
            document.save(byteArrayOutputStream);
-            return RedactionResult.builder()
-                    .document(byteArrayOutputStream.toByteArray())
-                    .numberOfPages(numberOfPages)
-                    .build();
+            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
        }

    }


    @Override
-    public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId,
-                                               @PathVariable("fileId") String fileId,
-                                               @RequestBody ManualRedactions manualRedactions) {
+    public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions) {

        var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
        log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
@@ -5,7 +5,6 @@ import org.springframework.beans.BeanUtils;

 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
-import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonIgnore;

 import lombok.AllArgsConstructor;
@@ -22,22 +21,20 @@ import lombok.SneakyThrows;
 public class RedTextPosition {

    private String textMatrix;
+    private float[] position;
+
+    @JsonIgnore
    private int rotation;
-    private float y;
+
+    @JsonIgnore
    private float pageHeight;
+
+    @JsonIgnore
    private float pageWidth;
+
    private String unicode;

-    @JsonAlias("xdirAdj")
-    @JsonAttribute(alternativeNames = {"xdirAdj"})
-    private float XDirAdj;
-
-    @JsonAlias("ydirAdj")
-    @JsonAttribute(alternativeNames = {"ydirAdj"})
-    private float YDirAdj;
-    private float width;
-    private float heightDir;
-    private float widthDirAdj;
+    @JsonIgnore
    private float dir;

    // not used in reanalysis
@@ -66,18 +63,37 @@ public class RedTextPosition {

        pos.setTextMatrix(textPosition.getTextMatrix().toString());

+        var position = new float[4];
+
+        position[0] = textPosition.getXDirAdj();
+        position[1] = textPosition.getYDirAdj();
+        position[2] = textPosition.getWidthDirAdj();
+        position[3] = textPosition.getHeightDir();
+
+        pos.setPosition(position);
        return pos;
    }


-    @JsonAlias("xdirAdj")
-    @JsonAttribute(alternativeNames = {"xdirAdj"})
-    public void setXDirAdj(float XDirAdj) {this.XDirAdj = XDirAdj;}
+    @JsonIgnore
+    public float getXDirAdj(){
+        return position[0];
+    }

+    @JsonIgnore
+    public float getYDirAdj(){
+        return position[1];
+    }

-    @JsonAlias("ydirAdj")
-    @JsonAttribute(alternativeNames = {"ydirAdj"})
-    public void setYDirAdj(float YDirAdj) {this.YDirAdj = YDirAdj;}
+    @JsonIgnore
+    public float getWidthDirAdj(){
+        return position[2];
+    }
+
+    @JsonIgnore
+    public float getHeightDir(){
+        return position[3];
+    }


 }
@@ -0,0 +1,62 @@
+package com.iqser.red.service.redaction.v1.server.parsing.model;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+import lombok.Getter;
+
+@Getter
+public enum TextDirection {
+    ZERO(0f),
+    QUARTER_CIRCLE(90f),
+    HALF_CIRCLE(180f),
+    THREE_QUARTER_CIRCLE(270f);
+
+    public static final String VALUE_STRING_SUFFIX = "°";
+
+    @JsonValue
+    private final float degrees;
+    private final float radians;
+
+    TextDirection(float degreeValue) {
+        degrees = degreeValue;
+        radians = (float) Math.toRadians(degreeValue);
+    }
+
+    @Override
+    public String toString() {
+        return degrees + VALUE_STRING_SUFFIX;
+    }
+
+    @com.dslplatform.json.JsonValue
+    public float jsonValue() {
+        return getDegrees();
+    }
+
+    @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+    public static TextDirection fromDegrees(float degrees) {
+
+        for (var dir : TextDirection.values()) {
+            if (degrees == dir.degrees) {
+                return dir;
+            }
+        }
+
+        throw new IllegalArgumentException(String.format("A value of %f is not supported by TextDirection", degrees));
+    }
+
+    public static TextDirection fromString(String degreesAsString) {
+
+        Objects.requireNonNull(degreesAsString, "Cannot construct a text direction from a null value");
+
+        String value = degreesAsString.strip();
+
+        if (degreesAsString.endsWith(VALUE_STRING_SUFFIX)) {
+            value = degreesAsString.replace(VALUE_STRING_SUFFIX + "$", "");
+        }
+
+        return fromDegrees(Float.parseFloat(value));
+    }
+}
@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.parsing.model;

+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -17,6 +19,7 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@@ -28,11 +31,14 @@ import lombok.extern.slf4j.Slf4j;
@JsonIgnoreProperties({"empty"})
 public class TextPositionSequence implements CharSequence {

+    public static final int HEIGHT_PADDING = 2;
    private int page;
    private List<RedTextPosition> textPositions = new ArrayList<>();

-    private float x1;
-    private float x2;
+    private TextDirection dir;
+    private int rotation;
+    private float pageHeight;
+    private float pageWidth;


    public TextPositionSequence(int page) {
@@ -41,7 +47,7 @@ public class TextPositionSequence implements CharSequence {
    }


-    public static TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
+    public TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {

        var textPositionSequence = new TextPositionSequence();
        textPositionSequence.textPositions = textPositions;
@@ -55,6 +61,10 @@ public class TextPositionSequence implements CharSequence {

        this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
        this.page = page;
+        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
    }


@@ -85,7 +95,15 @@ public class TextPositionSequence implements CharSequence {
    @Override
    public TextPositionSequence subSequence(int start, int end) {

-        return fromData(textPositions.subList(start, end), page);
+        var textPositionSequence = new TextPositionSequence();
+        textPositionSequence.textPositions = textPositions.subList(start, end);
+        textPositionSequence.page = page;
+        textPositionSequence.dir = dir;
+        textPositionSequence.rotation = rotation;
+        textPositionSequence.pageHeight = pageHeight;
+        textPositionSequence.pageWidth = pageWidth;
+
+        return textPositionSequence;
    }


@@ -106,15 +124,26 @@ public class TextPositionSequence implements CharSequence {
    }


-    public void add(RedTextPosition textPosition) {
+    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {

        this.textPositions.add(textPosition);
+        this.page = textPositionSequence.getPage();
+        this.dir = textPositionSequence.getDir();
+        this.rotation = textPositionSequence.getRotation();
+        this.pageHeight = textPositionSequence.getPageHeight();
+        this.pageWidth = textPositionSequence.getPageWidth();
    }


    public void add(TextPosition textPosition) {

        this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
+
+        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
+
    }


@@ -122,7 +151,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getX1() {

-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
            return textPositions.get(0).getYDirAdj() - getTextHeight();
        } else {
            return textPositions.get(0).getXDirAdj();
@@ -134,10 +163,10 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getX2() {

-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
            return textPositions.get(0).getYDirAdj();
        } else {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
+            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
        }
    }

@@ -146,15 +175,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getRotationAdjustedY() {

-        return textPositions.get(0).getY();
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getRotationAdjustedX() {
-
-        return textPositions.get(0).getXDirAdj();
+        return textPositions.get(0).getYDirAdj();
    }


@@ -162,10 +183,10 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getY1() {

-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
            return textPositions.get(0).getXDirAdj();
        } else {
-            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj();
+            return pageHeight - textPositions.get(0).getYDirAdj();
        }
    }

@@ -174,10 +195,10 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getY2() {

-        if (textPositions.get(0).getRotation() == 90) {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - 2;
+        if (rotation == 90) {
+            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - HEIGHT_PADDING;
        } else {
-            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() + getTextHeight();
+            return pageHeight - textPositions.get(0).getYDirAdj() + getTextHeight();
        }
    }

@@ -186,7 +207,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public float getTextHeight() {

-        return textPositions.get(0).getHeightDir() + 2;
+        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
    }


@@ -251,137 +272,39 @@ public class TextPositionSequence implements CharSequence {

    @JsonIgnore
    @JsonAttribute(ignore = true)
-    public int getRotation() {
-
-        return textPositions.get(0).getRotation();
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+    @SneakyThrows
    public Rectangle getRectangle() {

-        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, textPositions.get(0).getRotation(), textPositions.get(0).getDir());
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, rotation, dir);

-        float height = getTextHeight();
+        float textHeight = getTextHeight();

-        float posXInit = getX1();
-        float posXEnd;
-        float posYInit;
-        float posYEnd;
+        RedTextPosition firstTextPos = textPositions.get(0);
+        RedTextPosition lastTextPos = textPositions.get(textPositions.size() - 1);

-        if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 90f) {
-
-            posYInit = getX1();
-            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
-            posXInit = textPositions.get(0).getYDirAdj() + 2;
-            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
-
-        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 180f) {
-
-            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
-            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
-            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
-
-        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) {
-
-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
-
-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 0.0f) {
-
-            posXInit = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
-            posXEnd = textPositions.get(0).getXDirAdj();
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
-
-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 90.0f) {
-            posXEnd = textPositions.get(0).getYDirAdj() + 2;
-            posYInit = getY1();
-            posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
-
-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 180.0f) {
-
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getXDirAdj() - 4;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getXDirAdj();
-            posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1).getHeightDir();
-            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();
-
-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 270.0f) {
-
-            posXInit = textPositions.get(0).getPageWidth() - getX1();
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posYInit = textPositions.get(0).getPageHeight() - getY1();
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getXDirAdj() - height - 4;
-
-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 0f) {
-
-            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
-
-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 90f) {
-
-            posYInit = getX1();
-            posYEnd = getX2() - 3;
-            posXInit = textPositions.get(0).getYDirAdj() + 2;
-            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
-
-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
-
-            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
-            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
-            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
-
-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 270.0f) {
-
-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
-
-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) {
-
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = posYInit + 1;
-            posXInit = textPositions.get(0).getXDirAdj();
-            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
-
-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 90.0f) {
-
-            posYInit = getX1();
-            posYEnd = getX2() - height;
-            posXInit = textPositions.get(0).getYDirAdj() + 2;
-            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
-
-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 180.0f) {
-
-            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() - 4;
-            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
-            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
-
-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 270.0f) {
-
-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - height;
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
+        Point2D bottomLeft = new Point2D.Double(firstTextPos.getXDirAdj(), firstTextPos.getYDirAdj() - HEIGHT_PADDING);
+        Point2D topRight = new Point2D.Double(lastTextPos.getXDirAdj() + lastTextPos.getWidthDirAdj(), lastTextPos.getYDirAdj() + textHeight + HEIGHT_PADDING);

+        AffineTransform transform = new AffineTransform();
+        if (dir == TextDirection.ZERO || dir == TextDirection.HALF_CIRCLE) {
+            transform.rotate(dir.getRadians(), pageWidth / 2f, pageHeight / 2f);
+            transform.translate(0f, pageHeight + textHeight);
+            transform.scale(1., -1.);
+        } else if (dir == TextDirection.QUARTER_CIRCLE) {
+            transform.rotate(dir.getRadians(), pageWidth / 2f, pageWidth / 2f);
+            transform.translate(0f, pageWidth + textHeight);
+            transform.scale(1., -1.);
        } else {
-            // page rotation = 0 and text direction = 0
-            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
+            transform.rotate(dir.getRadians(), pageHeight / 2f, pageHeight / 2f);
+            transform.translate(0f, pageWidth + textHeight);
+            transform.scale(1., -1.);
        }

-        var rectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
-        log.debug("Rectangle: {}", rectangle);
-        return rectangle;
+        bottomLeft = transform.transform(bottomLeft, null);
+        topRight = transform.transform(topRight, null);
+
+        return new Rectangle( //
+                new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), page);
    }

 }
@@ -1,19 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import lombok.*;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
@Data
@Builder
@NoArgsConstructor
@@ -24,6 +19,7 @@ public class Entity implements ReasonHolder {
    private String word;
    private String type;
    private boolean redaction;
+    private boolean falsePositive;
    private String redactionReason;
    private String legalBasis;
    private List<EntityPositionSequence> positionSequences = new ArrayList<>();
@@ -9,7 +9,11 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati

 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Getter;

 public class SearchableText {

@@ -17,6 +21,7 @@ public class SearchableText {
    @JsonIgnore
    @JsonAttribute(ignore = true)
    private transient String stringRepresentation;
+    @Getter
    private final List<TextPositionSequence> sequences = new ArrayList<>();


@@ -100,7 +105,7 @@ public class SearchableText {
                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
                            .charAt(j, caseInsensitive) != ' ') {
-                        partMatch.add(searchSpace.get(i).textPositionAt(j));
+                        partMatch.add(searchSpace.get(i),searchSpace.get(i).textPositionAt(j));
                        if (!(j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
                                .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                            counter++;
@@ -194,7 +199,7 @@ public class SearchableText {
    }


-    public String buildString(List<TextPositionSequence> sequences) {
+    public static String buildString(List<TextPositionSequence> sequences) {

        StringBuilder sb = new StringBuilder();

@@ -218,6 +223,31 @@ public class SearchableText {
    }


+    public String getAsStringWithLinebreaksSorted() {
+
+
+        var sorted = sequences.stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
+
+        StringBuilder sb = new StringBuilder();
+
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : sorted) {
+
+            if (previous != null) {
+                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                    sb.append('\n');
+                } else {
+                    sb.append(' ');
+                }
+            }
+            sb.append(word.toString());
+            previous = word;
+        }
+        return sb.append("\n").toString();
+    }
+
+
+
    public String getAsStringWithLinebreaks() {

        StringBuilder sb = new StringBuilder();
@@ -78,7 +78,7 @@ public class Section {


    @SuppressWarnings("unused")
-    @WhenCondition
+    @ThenAction
    public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {

        Set<Entity> entitiesOfType = nerEntities.stream()
@@ -87,6 +87,7 @@ public class Section {
        List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
        Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());

        Set<Entity> finalResult = new HashSet<>();

@@ -112,7 +113,7 @@ public class Section {


    @SuppressWarnings("unused")
-    @WhenCondition
+    @ThenAction
    public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType,
                               @Argument(ArgumentType.TYPE) String combineTypes,
                               @Argument(ArgumentType.INTEGER) int maxDistanceBetween,
@@ -526,40 +527,54 @@ public class Section {
    }


+    @Deprecated
    @ThenAction
    @SuppressWarnings("unused")
-    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop,
-                              @Argument(ArgumentType.TYPE) String asType,
-                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                              @Argument(ArgumentType.STRING) String reason,
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason,
                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true);
    }


+    @Deprecated
    @ThenAction
    @SuppressWarnings("unused")
-    public void redactNotBetween(@Argument(ArgumentType.STRING) String start,
-                                 @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
-                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                 @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+    public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                 @Argument(ArgumentType.STRING) String reason) {

-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false);
    }


    @ThenAction
    @SuppressWarnings("unused")
-    public void redactLinesBetween(@Argument(ArgumentType.STRING) String start,
-                                   @Argument(ArgumentType.STRING) String stop,
-                                   @Argument(ArgumentType.TYPE) String asType,
-                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                   @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                                   @Argument(ArgumentType.STRING) String reason,
-                                   @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                 @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason) {
+
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                   @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

        redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
    }
@@ -567,11 +582,8 @@ public class Section {

    @ThenAction
    @SuppressWarnings("unused")
-    public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start,
-                                      @Argument(ArgumentType.STRING) String stop,
-                                      @Argument(ArgumentType.TYPE) String asType,
-                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                      @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+    public void redactNotLinesBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                      @Argument(ArgumentType.STRING) String reason) {

        redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
@@ -714,8 +726,7 @@ public class Section {
    @SuppressWarnings("unused")
    public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {

-        entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType()
-                .equals(EntityType.RECOMMENDATION));
+        entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.RECOMMENDATION));
    }


@@ -820,9 +831,25 @@ public class Section {

    @ThenAction
    @SuppressWarnings("unused")
-    public void redactSection(@Argument(ArgumentType.TYPE) String type,
-                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                              @Argument(ArgumentType.STRING) String reason,
+    public void redactSectionText(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+                                  @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                                 @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {

        for (SectionArea sectionArea : sectionAreas) {
@@ -855,8 +882,8 @@ public class Section {
    }


-    private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
-                                           int ruleNumber, String reason, String legalBasis, boolean redaction) {
+    private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis,
+                                           boolean redaction) {

        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
        Matcher matcher = compiledPattern.matcher(searchText);
@@ -886,13 +913,11 @@ public class Section {
            }
        });

-        return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
    }


-
-
-
    private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {

        entities.forEach(entity -> {
@@ -988,6 +1013,29 @@ public class Section {
    }


+
+    public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
+                                 String legalBasis) {
+
+
+        String[] values = StringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
+
+        if (values != null) {
+            for (String value : values) {
+                if (StringUtils.isNotBlank(value)) {
+                    Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+
+                    if (redactEverywhere && !isLocal()) {
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
+                    }
+                }
+            }
+        }
+    }
+
+
+
    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
                               String reason, String legalBasis, boolean redaction) {

@@ -1005,10 +1053,24 @@ public class Section {
    }


-    private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
-                               String reason, String legalBasis, boolean redaction) {
+    private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason, String legalBasis,
+                               boolean redaction) {

-        String[] values = StringUtils.substringsBetween(searchText, start, stop);
+        String[] values = new String[1];
+
+        if (start.isEmpty() && stop.isEmpty()) {
+            if (excludeHeadLine) {
+                values[0] = StringUtils.substringAfter(searchText, headline);
+            } else {
+                values[0] = searchText;
+            }
+        } else if (start.isEmpty() && searchText.contains(stop)) {
+            values[0] = StringUtils.substringBefore(searchText, stop);
+        } else if (stop.isEmpty() && searchText.contains(start)) {
+            values[0] = StringUtils.substringAfter(searchText, start);
+        } else {
+            values = StringUtils.substringsBetween(searchText, start, stop);
+        }

        if (values != null) {
            for (String value : values) {
@@ -21,6 +21,8 @@ public class ImageServiceResponse {
    @JsonAttribute(alternativeNames = {"imageMetadata"})
    private List<ImageMetadata> data = new ArrayList<>();

+    private List<ImageMetadata> dataCV = new ArrayList<>();
+
    @JsonProperty(value = "imageMetadata")
    @JsonAlias("data")
    @JsonAttribute(alternativeNames = {"imageMetadata"})
@@ -1,18 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
@@ -22,24 +9,21 @@ import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;

@Slf4j
@Service
@@ -51,7 +35,6 @@ public class EntityRedactionService {
    private final SurroundingWordsService surroundingWordsService;


-
    public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer,
                                     AnalyzeRequest analyzeRequest, NerEntities nerEntities) {

@@ -262,7 +245,8 @@ public class EntityRedactionService {
            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
        }

-        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions), nerFound);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
    }


@@ -5,7 +5,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;

 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
@@ -13,7 +12,6 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@@ -114,10 +112,7 @@ public class RedactionLogCreatorService {
                redactionLogEntry.setId(entityPositionSequence.getId());

                if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
-                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
-                            .stream()
-                            .flatMap(seq -> seq.getTextPositions().stream())
-                            .collect(Collectors.toList()), page);
+                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences());

                    redactionLogEntry.getPositions().addAll(rectanglesPerLine);

@@ -134,38 +129,43 @@ public class RedactionLogCreatorService {
    }


-    private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {
+    private List<Rectangle> getRectanglesPerLine(List<TextPositionSequence> textPositionSequences) {

        List<Rectangle> rectangles = new ArrayList<>();
-        if (textPositions.size() == 1) {
-            rectangles.add(TextPositionSequence.fromData(textPositions, page).getRectangle());
-        } else {
-            float x = textPositions.get(0).getXDirAdj();
-            float y = textPositions.get(0).getYDirAdj();
-            float width = textPositions.get(0).getWidthDirAdj();
-            float height = textPositions.get(0).getHeightDir();
-            int startIndex = 0;

-            for (int i = 1; i < textPositions.size(); i++) {
-                float xDirAdj = textPositions.get(i).getXDirAdj();
-                float yDirAdj = textPositions.get(i).getYDirAdj();
-                float widthDir = textPositions.get(i).getWidthDirAdj();
-                float heightDir = textPositions.get(i).getHeightDir();
+        if (textPositionSequences.size() == 1) {
+            rectangles.add(textPositionSequences.get(0).getRectangle());
+            return rectangles;
+        }

-                if (!(isCharInSameLine(y, yDirAdj, height, heightDir) && isCharClose(x, xDirAdj, width))) {
-                    rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, i), page)
-                            .getRectangle());
-                    y = yDirAdj;
-                    width = widthDir;
-                    height = heightDir;
-                    startIndex = i;
-                }
-                x = xDirAdj;
+        TextPositionSequence combinedSequence = new TextPositionSequence();
+        for (int i = 0; i < textPositionSequences.size(); i++) {
+
+            if(combinedSequence.getTextPositions().isEmpty()){
+                combinedSequence = textPositionSequences.get(i);
+                continue;
            }
-            if (startIndex != textPositions.size()) {
-                rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page)
-                        .getRectangle());
+
+            float lastSeqX = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getXDirAdj();
+            float lastSeqY = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getYDirAdj();
+            float lastSeqWidth = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getWidthDirAdj();
+            float lastSeqHeight = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getHeightDir();
+
+            float currentSeqX = textPositionSequences.get(i).getTextPositions().get(0).getXDirAdj();
+            float currentSeqY = textPositionSequences.get(i).getTextPositions().get(0).getYDirAdj();
+            float currentSeqHeight = textPositionSequences.get(i).getTextPositions().get(0).getHeightDir();
+
+            if (isCharInSameLine(lastSeqY, currentSeqY, lastSeqHeight, currentSeqHeight) && isCharClose(lastSeqX, currentSeqX, lastSeqWidth)) {
+                combinedSequence.getTextPositions().addAll(textPositionSequences.get(i).getTextPositions());
+            } else {
+                rectangles.add(combinedSequence.getRectangle());
+                combinedSequence = textPositionSequences.get(i);
            }
+
+        }
+
+        if(!combinedSequence.getTextPositions().isEmpty()) {
+            rectangles.add(combinedSequence.getRectangle());
        }

        return rectangles;
@@ -207,7 +207,8 @@ public class RedactionLogCreatorService {
                .redacted(entity.isRedaction())
                .isHint(isHint(entity.getType(), dossierTemplateId))
                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
+                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType()
+                        .equals(EntityType.FALSE_RECOMMENDATION))
                .section(entity.getHeadline())
                .sectionNumber(entity.getSectionNumber())
                .matchedRule(entity.getMatchedRule())
@@ -25,6 +25,8 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.entity
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.Change;
+import com.iqser.red.service.redaction.v1.model.ChangeType;
 import com.iqser.red.service.redaction.v1.model.ManualChange;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.Point;
@@ -100,7 +102,7 @@ public class RedactionLogMergeService {
        log.info("Merging Redaction log with manual redactions");
        if (manualRedactions != null) {

-            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
+            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types, redactionLog.getAnalysisNumber());

            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);

@@ -372,14 +374,14 @@ public class RedactionLogMergeService {

    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
                                                       Map<String, List<Comment>> comments, Colors colors,
-                                                       List<Type> types) {
+                                                       List<Type> types, int analysisNumber) {

        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();

        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {

            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types);
+                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types, analysisNumber);
                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
                redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
@@ -420,7 +422,7 @@ public class RedactionLogMergeService {


    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
-                                                      Colors colors, List<Type> types) {
+                                                      Colors colors, List<Type> types, int analysisNumber) {

        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();

@@ -445,6 +447,7 @@ public class RedactionLogMergeService {
                .sectionNumber(-1)
                .rectangle(manualRedactionEntry.isRectangle())
                .manualChanges(changeList)
+                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
                .build();
    }

@@ -1,19 +1,35 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.server.classification.model.*;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;

-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
+import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
@@ -28,15 +44,10 @@ public class SectionTextBuilderService {

            List<Table> tables = paragraph.getTables();
            for (Table table : tables) {
-                if (table.getColCount() == 2) {
-                    sectionTexts.add(processTableAsOneText(table, sectionNumber));
-                } else {
-                    sectionTexts.addAll(processTablePerRow(table, sectionNumber));
-                }
+                sectionTexts.addAll(processTablePerRow(table, sectionNumber));
                sectionNumber.incrementAndGet();
            }
-            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph
-                    .getImages()));
+            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
            sectionNumber.incrementAndGet();
        }

@@ -62,58 +73,50 @@ public class SectionTextBuilderService {
    private List<SectionText> processTablePerRow(Table table, AtomicInteger sectionNumber) {

        List<SectionText> sectionTexts = new ArrayList<>();
-        for (List<Cell> row : table.getRows()) {
-            SearchableText searchableRow = new SearchableText();
-            Map<String, CellValue> tabularData = new HashMap<>();
-            int start = 0;
-            List<Integer> cellStarts = new ArrayList<>();
-            SectionText sectionText = new SectionText();
-            for (Cell cell : row) {
+        boolean hasHeader = hasTableHeader(table);

+        for (List<Cell> row : table.getRows()) {
+
+            List<TextBlock> textBlocks = new ArrayList<>();
+            List<SectionArea> areas = new ArrayList<>();
+            Map<String, CellValue> tabularData = new HashMap<>();
+            List<Integer> startOffsets = new ArrayList<>();
+
+            int startOffset = 0;
+            for (int cellNum = 0; cellNum < row.size(); cellNum++) {
+
+                Cell cell = row.get(cellNum);
                if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
                    continue;
                }

-                SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
-                        .getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
-                        .get(0)
-                        .getSequences()
-                        .get(0)
-                        .getPage(), null);
-                sectionText.getSectionAreas().add(sectionArea);
-                sectionText.getTextBlocks().addAll(cell.getTextBlocks());
-
-                int cellStart = start;
+                SectionArea sectionArea = getSectionArea(cell);
+                areas.add(sectionArea);

                if (!cell.isHeaderCell()) {
-                    cell.getHeaderCells().forEach(headerCell -> {
-                        StringBuilder headerBuilder = new StringBuilder();
-                        headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
-                        String headerName = headerBuilder.toString()
-                                .replaceAll("\n", "")
-                                .replaceAll(" ", "")
-                                .replaceAll("-", "");
+                    String headerName = getHeaderName(hasHeader, cell, table.getRows().get(0), row.size(), cellNum);
+                    if (headerName != null) {
                        sectionArea.setHeader(headerName);
-                        tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
-                    });
+                        tabularData.put(headerName, new CellValue(cell.getTextBlocks(), startOffset));
+                    }
                }

-                for (TextBlock textBlock : cell.getTextBlocks()) {
-                    // TODO avoid cell overlap merging.
-                    searchableRow.addAll(textBlock.getSequences());
-                }
-                cellStarts.add(cellStart);
-                start = start + cell.toString().trim().length() + 1;
+                textBlocks.addAll(cell.getTextBlocks());

+                startOffsets.add(startOffset);
+                startOffset = startOffset + cell.toString().trim().length() + 1;
            }

-            sectionText.setText(searchableRow.toString());
-            sectionText.setHeadline(table.getHeadline());
-            sectionText.setSectionNumber(sectionNumber.intValue());
-            sectionText.setTable(true);
-            sectionText.setTabularData(tabularData);
-            sectionText.setCellStarts(cellStarts);
-            sectionTexts.add(sectionText);
+            sectionTexts.add(SectionText.builder()
+                    .text(getRowText(textBlocks))
+                    .headline(table.getHeadline())
+                    .sectionNumber(sectionNumber.intValue())
+                    .isTable(true)
+                    .tabularData(tabularData)
+                    .cellStarts(startOffsets)
+                    .textBlocks(textBlocks)
+                    .sectionAreas(areas)
+                    .build());

            sectionNumber.incrementAndGet();
        }
@@ -122,56 +125,53 @@ public class SectionTextBuilderService {
    }


-    private SectionText processTableAsOneText(Table table, AtomicInteger sectionNumber) {
+    public String getRowText(List<TextBlock> rowTextBlocks) {

-        SearchableText entireTableText = new SearchableText();
-        SectionText sectionText = new SectionText();
-        int start = 0;
-        List<Integer> cellStarts = new ArrayList<>();
-        for (List<Cell> row : table.getRows()) {
-
-
-            for (Cell cell : row) {
-                if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
-                    continue;
-                }
-
-                SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
-                        .getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
-                        .get(0)
-                        .getSequences()
-                        .get(0)
-                        .getPage(), null);
-                sectionText.getTextBlocks().addAll(cell.getTextBlocks());
-                sectionText.getSectionAreas().add(sectionArea);
-
-                for (TextBlock textBlock : cell.getTextBlocks()) {
-                    entireTableText.addAll(textBlock.getSequences());
-                }
-
-
-                cellStarts.add(start);
-                start = start + cell.toString().trim().length() + 1;
-            }
-
-        }
-        sectionText.setCellStarts(cellStarts);
-
-        sectionText.setText(entireTableText.toString());
-        sectionText.setHeadline(table.getHeadline());
-        sectionText.setSectionNumber(sectionNumber.intValue());
-        sectionText.setTable(true);
-        return sectionText;
+        return SearchableText.buildString(rowTextBlocks.stream().map(textBlock -> textBlock.getSequences()).flatMap(List::stream).collect(Collectors.toList()));
    }


-    private SectionText processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks, String headline,
-                                    AtomicInteger sectionNumber, List<PdfImage> images) {
+    private boolean hasTableHeader(Table table) {
+
+        return table.getRows().stream().anyMatch(row -> row.stream().anyMatch(cell -> !cell.isHeaderCell() && !cell.getHeaderCells().isEmpty()));
+    }
+
+
+    private SectionArea getSectionArea(Cell cell) {
+
+        return new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell.getWidth(), (float) cell.getHeight(),
+                cell.getTextBlocks().get(0).getSequences().get(0).getPage(), null);
+    }
+
+
+    private String getHeaderName(boolean hasHeader, Cell currentCell, List<Cell> firstRow, int currentRowSize, int cellNum) {
+
+        if (hasHeader && !currentCell.getHeaderCells().isEmpty()) {
+            return buildHeaderName(currentCell.getHeaderCells().get(currentCell.getHeaderCells().size() - 1));
+        }
+
+        if (!hasHeader && currentRowSize == firstRow.size() && currentCell != firstRow.get(cellNum)) {
+            return buildHeaderName(firstRow.get(cellNum));
+        }
+
+        return null;
+    }
+
+
+    private String buildHeaderName(Cell cell) {
+
+        StringBuilder headerBuilder = new StringBuilder();
+        cell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
+        return headerBuilder.toString().replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
+    }
+
+
+    private SectionText processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks, String headline, AtomicInteger sectionNumber, List<PdfImage> images) {

        SectionText sectionText = new SectionText();
        for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
-            SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock
-                    .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage(), null);
+            SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock.getWidth(),
+                    paragraphTextBlock.getHeight(), paragraphTextBlock.getPage(), null);
            sectionText.getSectionAreas().add(sectionArea);
        }

@@ -179,9 +179,7 @@ public class SectionTextBuilderService {
        sectionText.setHeadline(headline);
        sectionText.setSectionNumber(sectionNumber.intValue());
        sectionText.setTable(false);
-        sectionText.setImages(images.stream()
-                .map(image -> convertImage(image, sectionNumber.intValue(), headline))
-                .collect(Collectors.toSet()));
+        sectionText.setImages(images.stream().map(image -> convertImage(image, sectionNumber.intValue(), headline)).collect(Collectors.toSet()));
        sectionText.setTextBlocks(paragraphTextBlocks);
        return sectionText;
    }
@@ -190,9 +188,7 @@ public class SectionTextBuilderService {
    private Image convertImage(PdfImage pdfImage, int sectionNumber, String headline) {

        return Image.builder()
-                .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType()
-                        .name()
-                        .toLowerCase(Locale.ROOT))
+                .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType().name().toLowerCase(Locale.ROOT))
                .position(pdfImage.getPosition())
                .sectionNumber(sectionNumber)
                .section(headline)
@@ -1,28 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

+import java.util.*;
+import java.util.stream.Collectors;
+
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
@@ -39,12 +26,10 @@ public class EntitySearchUtils {

        if (details.getEntityType() == EntityType.RECOMMENDATION) {
            Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
-            removeFalsePositives(found, falseRecommendations);
-            found.addAll(falseRecommendations);
+            markFalsePositives(found, falseRecommendations);
        } else {
            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
-            removeFalsePositives(found, falsePositives);
-            found.addAll(falsePositives);
+            markFalsePositives(found, falsePositives);
        }

        return found;
@@ -93,7 +78,7 @@ public class EntitySearchUtils {
            }
        }

-        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()){
+        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()) {
            applyResizeRedactions(entities, manualRedactions);
        }

@@ -131,7 +116,7 @@ public class EntitySearchUtils {
    }


-    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
+    public void markFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {

        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : falsePositives) {
@@ -142,8 +127,12 @@ public class EntitySearchUtils {
                }
            }
        }
-        wordsToRemove.forEach(entities::remove);
-        entities.removeAll(falsePositives);
+        wordsToRemove.forEach(e -> e.setFalsePositive(true));
+        for (var entity : entities) {
+            if (falsePositives.contains(entity)) {
+                entity.setFalsePositive(true);
+            }
+        }
    }


@@ -152,13 +141,22 @@ public class EntitySearchUtils {
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
            for (Entity inner : entities) {
+//                // skip cross-type false positives
+//                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
+//                    continue;
+//                }
+//                // skip cross-type false reccommendations
+//                if(word.getEntityType() == EntityType.FALSE_RECOMMENDATION && inner.getEntityType() == EntityType.RECOMMENDATION && !inner.getType().equals(word.getType())){
+//                    continue;
+//                }
+
                if (inner.getWord().length() < word.getWord()
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                        wordsToRemove.add(word);
-                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
                            inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
-                        if(inner.isResized()){
+                        if (inner.isResized()) {
                            wordsToRemove.add(word);
                        } else {
                            wordsToRemove.add(inner);
@@ -27,26 +27,48 @@ public class ImageService {
    private final ObjectMapper objectMapper;
    private final RedactionStorageService redactionStorageService;

+
    @SneakyThrows
-    public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId){
+    public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId) {

        var imageClassificationStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));

-
        ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class);

        Map<Integer, List<PdfImage>> images = new HashMap<>();
        imageServiceResponse.getData().forEach(imageMetadata -> {
-            var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)) : ImageType.OTHER;
-            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
-                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), classification,imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber()));
+            var classification = imageMetadata.getFilters()
+                    .isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification()
+                    .getLabel()
+                    .toUpperCase(Locale.ROOT)) : ImageType.OTHER;
+            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
+                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition()
+                            .getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry()
+                            .getWidth(), imageMetadata.getGeometry()
+                            .getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition()
+                            .getPageNumber()));
+        });
+
+        // Currently This is a copy but, it will be changed later because i don' t think that we should unclassified images.
+        imageServiceResponse.getDataCV().forEach(imageMetadata -> {
+            var classification = imageMetadata.getFilters()
+                    .isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification()
+                    .getLabel()
+                    .toUpperCase(Locale.ROOT)) : ImageType.OTHER;
+            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
+                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition()
+                            .getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry()
+                            .getWidth(), imageMetadata.getGeometry()
+                            .getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition()
+                            .getPageNumber()));
        });

        return images;
    }


-    public void findOcr(Page page){
+    public void findOcr(Page page) {
+
        page.getImages().forEach(image -> {
            if (image.getImageType().equals(ImageType.OTHER)) {
                page.getTextBlocks().forEach(textblock -> {
@@ -59,5 +81,4 @@ public class ImageService {
        });
    }

-
 }
@@ -1,8 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.storage;

-import com.dslplatform.json.DslJson;
-import com.dslplatform.json.runtime.Settings;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.InputStream;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
@@ -12,16 +13,12 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
+
 import io.micrometer.core.annotation.Timed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.stereotype.Service;
-
-import java.io.IOException;
-import java.io.InputStream;

@Slf4j
@Service
@@ -13,13 +13,20 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
 public abstract class AbstractTextContainer {

+    @JsonIgnore
    protected float minX;
+    @JsonIgnore
    protected float maxX;
+    @JsonIgnore
    protected float minY;
+    @JsonIgnore
    protected float maxY;
+    @JsonIgnore
    protected String classification;
+    @JsonIgnore
    protected int page;

+    @JsonIgnore
    private Orientation orientation = Orientation.NONE;

    public abstract String getText();
@@ -1,14 +1,5 @@
 package com.iqser.red.service.redaction.v1.server;

-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-import lombok.SneakyThrows;
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -19,6 +10,16 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

+import org.apache.commons.io.IOUtils;
+import org.springframework.core.io.InputStreamResource;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
 public class FileSystemBackedStorageService implements StorageService {

    private final Map<String, File> dataMap = new HashMap<>();
@@ -62,11 +63,8 @@ public class FileSystemBackedStorageService implements StorageService {
    }

    private ObjectMapper getMapper() {
-        var objectMapper = new ObjectMapper();
-        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        objectMapper.registerModule(new JavaTimeModule());
-        objectMapper.findAndRegisterModules();
-        return objectMapper;
+
+        return ObjectMapperFactory.create();
    }

    @Override
@@ -41,6 +41,7 @@ import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
+import org.mockito.stubbing.Answer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -90,6 +91,7 @@ public class RedactionIntegrationTest {
    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
    private static final String IMPORTED_REDACTION = "imported_redaction";
    private static final String PII = "PII";
+    private static final String ROTATE_SIMPLE = "RotateSimple";

    @Autowired
    private RedactionController redactionController;
@@ -217,25 +219,26 @@ public class RedactionIntegrationTest {

    private void mockDictionaryCalls(Long version) {

-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
-        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
-        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
-        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false));
-        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMAGE, false));
-        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(OCR, false));
-        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(LOGO, false));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SIGNATURE, false));
-        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION, false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD, false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE, false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS, true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION, true));

    }

@@ -327,6 +330,36 @@ public class RedactionIntegrationTest {
    }


+
+    @Test
+    public void titleExtraction() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/32 - Emamectin Benzoate Technical - Acute Oral Toxicity - Mouse.pdf");
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
+
+
    @Test
    @Ignore
    @SneakyThrows
@@ -851,6 +884,63 @@ public class RedactionIntegrationTest {
        System.out.println("numberOfPages: " + result.getNumberOfPages());
    }

+    @Test
+    public void testRotationsSimple() throws IOException {
+        loadOnlyDictionaryForSimpleFile();
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        System.out.println("testTableRedaction");
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/new/RotateTestFileSimple.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/AnnotatedSimple.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
+    @Test
+    public void testTableHeader() throws IOException {
+
+        System.out.println("testTableHeader");
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/Minimal Examples/NoHeaderTable.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(5);
+
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+

    @Test
    public void testFindDictionaryEntryInResizedEntryPosition() throws IOException {
@@ -1004,7 +1094,7 @@ public class RedactionIntegrationTest {
    public void classificationTest() throws IOException {

        System.out.println("classificationTest");
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");

        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());

@@ -1474,6 +1564,15 @@ public class RedactionIntegrationTest {

    }

+    private void loadOnlyDictionaryForSimpleFile() {
+        dictionary.clear();
+        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+    }
+

    private static String loadFromClassPath(String path) {

@@ -1532,6 +1631,7 @@ public class RedactionIntegrationTest {
        typeColorMap.put(FORMULA, "#ffe187");
        typeColorMap.put(SIGNATURE, "#ffe187");
        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
+        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");

        hintTypeMap.put(VERTEBRATE, true);
        hintTypeMap.put(ADDRESS, false);
@@ -1552,6 +1652,7 @@ public class RedactionIntegrationTest {
        hintTypeMap.put(SIGNATURE, false);
        hintTypeMap.put(DOSSIER_REDACTIONS, false);
        hintTypeMap.put(IMPORTED_REDACTION, false);
+        hintTypeMap.put(ROTATE_SIMPLE, false);

        caseInSensitiveMap.put(VERTEBRATE, true);
        caseInSensitiveMap.put(ADDRESS, false);
@@ -1572,6 +1673,7 @@ public class RedactionIntegrationTest {
        caseInSensitiveMap.put(FORMULA, true);
        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
+        caseInSensitiveMap.put(ROTATE_SIMPLE, true);

        recommendationTypeMap.put(VERTEBRATE, false);
        recommendationTypeMap.put(ADDRESS, false);
@@ -1592,6 +1694,7 @@ public class RedactionIntegrationTest {
        recommendationTypeMap.put(LOGO, false);
        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
        recommendationTypeMap.put(IMPORTED_REDACTION, false);
+        recommendationTypeMap.put(ROTATE_SIMPLE, false);

        rankTypeMap.put(PURITY, 155);
        rankTypeMap.put(PII, 150);
@@ -1612,6 +1715,7 @@ public class RedactionIntegrationTest {
        rankTypeMap.put(FORMULA, 26);
        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
        rankTypeMap.put(IMPORTED_REDACTION, 200);
+        rankTypeMap.put(ROTATE_SIMPLE, 150);

        colors.setSkippedColor("#cccccc");
        colors.setRequestAddColor("#04b093");
@@ -1661,7 +1765,6 @@ public class RedactionIntegrationTest {
                .build();
    }

-
    private String cleanDictionaryEntry(String entry) {

        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
@@ -1669,6 +1772,8 @@ public class RedactionIntegrationTest {


    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+        if (entries == null)
+            entries = Collections.emptyList();

        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
        entries.forEach(entry -> {
@@ -27,7 +27,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
@@ -86,7 +85,9 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;

 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;

+@Slf4j
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RulesTest.RulesTestConfiguration.class)
@@ -114,10 +115,13 @@ public class RulesTest {
    private static final String IMPORTED_REDACTION = "imported_redaction";
    private static final String PII = "PII";
    private static final String RESOURCES_PATH = "src/test/resources/";
-    private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
+    private static final String REDACTION_LOG_PATH = "RedactionLog/";
+    private static final String REDACTION_LOG_RESOURCES_PATH = RESOURCES_PATH + REDACTION_LOG_PATH;
    private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
    private final static String TEST_DOSSIER_ID = "123";
    private String TEST_FILE_ID = "123";
+    private int counter = 0;
+    private int fileSize = 0;
    private final Map<String, List<String>> dictionary = new HashMap<>();
    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
    private final Map<String, List<String>> falsePositive = new HashMap<>();
@@ -176,7 +180,9 @@ public class RulesTest {
                .hexColor("#ffe187")
                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .build()));

        mockDictionaryCalls(null);
        mockDictionaryCalls(0L);
@@ -196,7 +202,7 @@ public class RulesTest {

    /**
     * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
-     * If the RedactionLog already exists, the generating will be skipped
+     * If the RedactionLog already exists, it will be overwritten
     * Test is ignored, because it's for manual tests.
     */
    @Ignore
@@ -204,20 +210,22 @@ public class RulesTest {
    public void generateRedactionLogForOneFile() {

        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
-        generateAndSaveRedactionLog(fileName);
+        fileSize = 1;
+        generateAndSaveRedactionLog(fileName, true);
    }


    /**
     * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
-     * If a RedactionLog already exists, the generating will be skipped
+     * If a RedactionLog already exists, the generating for this will be skipped
+     * Please commit generated files, if not the following tests will fail
     */
-    @IfProfileValue(name = "test-groups", value = "rules-test")
    @Test
    public void generateRedactionLogForAllFiles() {

        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
        System.out.println("Will generate RedactionLog for " + files.size() + " files.");
+        fileSize = files.size();
        files.forEach(this::generateAndSaveRedactionLog);
    }

@@ -225,41 +233,69 @@ public class RulesTest {
    /**
     * Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
     * If RedactionLog Json does not exist, test will fail.
-     * Test is ignored, because it's for manual tests.
     */
-    @Ignore
    @Test
    public void analyseFileAndCompareRedactionLog() {

-        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
+        String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf";
+        fileSize = 1;
        analyseFileAndCompareRedactionLog(fileName);
    }


    /**
     * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
-     * If RedactionLogs Json does not exist, test will fail.
+     * If RedactionLog Json for one file does not exist, whole test will fail.
     */
    @IfProfileValue(name = "test-groups", value = "rules-test")
    @Test
    public void analyseAllFilesAndCompareRedactionLogs() {

-        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
-        System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
-        files.forEach(this::analyseFileAndCompareRedactionLog);
+        Set<String> fileNames = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
+        System.out.println("Will analyse " + fileNames.size() + " files and compare its RedactionLogs.");
+        fileSize = fileNames.size();
+
+        Map<String, AssertionError> failedFiles = new HashMap<>();
+
+        for (String fileName : fileNames) {
+
+            try {
+                analyseFileAndCompareRedactionLog(fileName);
+            } catch (AssertionError assertionError) {
+                failedFiles.put(fileName, assertionError);
+            }
+        }
+
+        if (!failedFiles.isEmpty()) {
+            log.warn("WARNING: {} files from {} failed", failedFiles.size(), fileSize);
+            for (String fileName : failedFiles.keySet()) {
+                log.warn(" - '{}' failed with Error: {} See line {} in {}", fileName, failedFiles.get(fileName), failedFiles.get(fileName)
+                        .getStackTrace()[0].getLineNumber(), failedFiles.get(fileName).getStackTrace()[0].getClassName());
+            }
+        }
+
+        assertThat(failedFiles).isEmpty();
+
+    }
+
+
+    private void generateAndSaveRedactionLog(String fileName) {
+
+        generateAndSaveRedactionLog(fileName, false);
    }


    @SneakyThrows
-    private void generateAndSaveRedactionLog(String fileName) {
+    private void generateAndSaveRedactionLog(String fileName, boolean force) {

-        if (doesRedactionLogExist(fileName)) {
-            System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
+        counter++;
+        if (!force && doesRedactionLogExist(fileName)) {
+            System.out.println("(" + counter + "/" + fileSize + ") Skip generating RedactionLog as Json for " + fileName + " because it already exists.");

        } else {
            generateTestFileId(fileName);

-            System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+            System.out.println("(" + counter + "/" + fileSize + ") Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);

            loadNerForTest();

@@ -278,8 +314,9 @@ public class RulesTest {
    @SneakyThrows
    public void analyseFileAndCompareRedactionLog(String fileName) {

+        counter++;
        generateTestFileId(fileName);
-        System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
+        System.out.println("(" + counter + "/" + fileSize + ") Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");

        RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);

@@ -299,7 +336,11 @@ public class RulesTest {
        assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
        assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());

-        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
+        assertThat(redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(r -> !r.isFalsePositive())
+                .collect(Collectors.toSet())
+                .size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size());
        assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());

        for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
@@ -387,7 +428,7 @@ public class RulesTest {
                assertThat(savedManualChange).isPresent();
            }

-            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
+            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(new Engine[0]));

            assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
            assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
@@ -399,7 +440,7 @@ public class RulesTest {
    private boolean doesRedactionLogExist(String pdfFileName) {

        File pdfFile = new File(pdfFileName);
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
        File file = new File(directory, fileName);
        return file.exists();
@@ -421,7 +462,7 @@ public class RulesTest {

        File pdfFile = new File(pdfFileName);

-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
        File dr = new File(directory);
        boolean created = dr.mkdirs();
        if (created) {
@@ -441,8 +482,20 @@ public class RulesTest {
    private RedactionLog loadSavedRedactionLog(String pdfFileName) {

        File pdfFile = new File(pdfFileName);
-        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String directory = cleanPath(pdfFile.getParentFile().getPath());
+        if (StringUtils.contains(directory, RESOURCES_PATH)) {
+            if (!StringUtils.endsWith(directory, "/")) {
+                directory = directory + "/";
+            }
+            directory = directory + REDACTION_LOG_PATH;
+        } else {
+            if (StringUtils.startsWith(directory, "/")) {
+                directory = StringUtils.substring(directory, 1);
+            }
+            directory = REDACTION_LOG_RESOURCES_PATH + directory;
+        }
        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+
        File file = new File(directory, fileName);
        assertThat(file).exists();

@@ -450,6 +503,31 @@ public class RulesTest {
    }


+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = cleanPath(path.toAbsolutePath().toString());
+                    int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length();
+
+                    fileNames.add(StringUtils.substring(absolutePath, pos));
+                }
+            }
+        }
+        return fileNames;
+    }
+
+
+    private String cleanPath(String path) {
+
+        return StringUtils.replace(path, "\\", "/");
+    }
+
+
    @SneakyThrows
    private void loadNerForTest() {

@@ -588,24 +666,6 @@ public class RulesTest {
    }


-    @SneakyThrows
-    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
-
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
-            for (Path path : stream) {
-                if (path.toFile().isDirectory()) {
-                    getFileNames(fileNames, path);
-                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
-                    String absolutePath = path.toAbsolutePath().toString();
-                    int pos = StringUtils.indexOf(absolutePath, StringUtils.replace(RESOURCES_PATH, "/", "\\")) + 18;
-                    fileNames.add(StringUtils.substring(absolutePath, pos));
-                }
-            }
-        }
-        return fileNames;
-    }
-
-
    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {

        return Type.builder()
@@ -1,14 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.annotate;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.*;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import lombok.RequiredArgsConstructor;
+import java.awt.Color;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -22,12 +23,21 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;

-import java.awt.*;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.CellRectangle;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.SectionRectangle;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;

@Service
@RequiredArgsConstructor
@@ -0,0 +1,71 @@
+package com.iqser.red.service.redaction.v1.server.parsing.model;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.pdfbox.util.Matrix;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.storage.commons.properties.StorageProperties;
+import com.iqser.red.storage.commons.service.ObjectSerializer;
+
+import lombok.SneakyThrows;
+
+public class TextPositionSequenceTest {
+
+    private static final String TEXT_POSITION_SEQUENCE_AS_JSON = "{\n" //
+            + "  \"page\": 1,\n" //
+            + "  \"textPositions\": [],\n" //
+            + "  \"dir\": 180.0,\n" //
+            + "  \"rotation\": 0,\n" //
+            + "  \"pageHeight\": 800,\n" //
+            + "  \"pageWidth\": 600\n"  //
+            + "}";
+
+    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties());
+
+
+    @Test
+    @SneakyThrows
+    public void testDeserializationWithJackson() {
+
+        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(
+                new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+                TextPositionSequence.class);
+
+        assertPropertiesAfterJsonDeserialization(textPositionSequence);
+    }
+
+
+    private void assertPropertiesAfterJsonDeserialization(TextPositionSequence textPositionSequence) {
+
+        assertThat(textPositionSequence.getPage()).isEqualTo(1);
+        assertThat(textPositionSequence.getTextPositions()).hasSize(0);
+        assertThat(textPositionSequence.getDir()).isEqualTo(TextDirection.HALF_CIRCLE);
+        assertThat(textPositionSequence.getRotation()).isEqualTo(0);
+        assertThat(textPositionSequence.getPageHeight()).isEqualTo(800f);
+        assertThat(textPositionSequence.getPageWidth()).isEqualTo(600f);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testDeserializationWithDslJson() {
+
+        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithDslJson(
+                new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+                TextPositionSequence.class);
+
+        assertPropertiesAfterJsonDeserialization(textPositionSequence);
+    }
+
+
+    private Matrix createIdentityMatrix() {
+
+        return new Matrix();
+    }
+
+}
@@ -5,9 +5,11 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

 import java.time.OffsetDateTime;
@@ -19,6 +21,9 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest

    public static final String FILE_NAME = "test-file";

+    @Autowired
+    private AnnotationService annotationService;
+
    @Test
    @SneakyThrows
    public void testFile() {
@@ -81,7 +86,19 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
        redactionMessageReceiver.receiveAnalyzeRequest(in, false);


-        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        var redactionLog = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        redactionLog.getRedactionLogEntry().forEach(entry ->{
+//            log.info("Entry {}", entry);
+//        });
+//
+//        var annotated = annotationService.annotate(AnnotateRequest.builder().fileId("fileId").dossierId("dossierId").build());
+//        annotated.getDocument();
+//
+//        File tempFile = File.createTempFile("annotated",".pdf");
+//        IOUtils.write(annotated.getDocument(), new FileOutputStream(tempFile));
+//        log.warn("File saved to: {}",tempFile.getAbsolutePath());
+//        Runtime.getRuntime().exec("open "+tempFile.getAbsolutePath());
    }

+
 }
@@ -1,17 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.realdata;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.nullable;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.*;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -30,27 +34,14 @@ import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.test.context.junit4.SpringRunner;

-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
-import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;

-import lombok.SneakyThrows;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -143,6 +134,7 @@ public class LiveDataIntegrationTest {
            if (found.isPresent()) {
                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
                type.setEntries(getEntries(typeName, type.getTypeId()));
+                type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId()));

                return type;
            } else {
@@ -196,7 +188,22 @@ public class LiveDataIntegrationTest {

        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
        for (var resource : dictionaryResources) {
-            if (Objects.requireNonNull(resource.getFilename()).contains(typeName)) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) {
+
+                List<String> lines = IOUtils.readLines(resource.getInputStream());
+                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
+
+            }
+        }
+        return new ArrayList<>();
+    }
+
+    @SneakyThrows
+    private List<DictionaryEntry> getFalsePositiveEntries(String typeName, String typeId) {
+
+        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
+        for (var resource : dictionaryResources) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) {

                List<String> lines = IOUtils.readLines(resource.getInputStream());
                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
@@ -1,15 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import org.apache.commons.lang3.StringUtils;
-import org.junit.Test;
-
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;

 public class RegExPatternTest {

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"12ac3d26ce82a77e469c64d56081662d","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Table in: COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.8,"y":651.16003},"width":54.039078,"height":-11.520004,"page":4}],"sectionNumber":3,"textBefore":"Report: Document F/02, ","textAfter":". (2014). Cyprodinil","comments":[],"startOffset":23,"endOffset":35,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:36:30.044122+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dd614353319e7c5a8cf8b3346dc12fac","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":306.84,"y":679.12},"width":59.71997,"height":-12.0,"page":4}],"sectionNumber":5,"textBefore":"F report (","textAfter":"., 2014).","comments":[],"startOffset":247,"endOffset":259,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:36:30.044134+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"36a5e8909a18a5080dd423afb8efd7a7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":171.20801,"y":320.0},"width":-12.208008,"height":-20.99997,"page":1}],"sectionNumber":3,"textBefore":"Product Metabolism and ","textAfter":" 1 of","comments":[],"startOffset":368,"endOffset":372,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:43:43.751956+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dca119c3aadd348bc43c0b700c0075c3","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":112.20801,"y":81.0},"width":-12.208008,"height":-21.0,"page":1}],"sectionNumber":3,"textBefore":"Pages RAM 465/02 ","textAfter":" 1 Tel:","comments":[],"startOffset":399,"endOffset":403,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:43:43.751958+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"ef6e0f0ed413dbd7299d992d9a50840b","type":"CBI_author","value":"Crook","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":386.5152,"y":389.0},"width":-12.515198,"height":-26.024994,"page":1}],"sectionNumber":5,"textBefore":"Author : SJ ","textAfter":" Analytical Science","comments":[],"startOffset":12,"endOffset":17,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:43:43.751958+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"0fac36714dd9d1c745565c6dfe06b9b7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":514.14215,"y":59.532837},"width":23.39972,"height":-11.591187,"page":2}],"sectionNumber":4,"textBefore":"RAM 465/02 ","textAfter":" 2 Summary","comments":[],"startOffset":11,"endOffset":15,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:43:43.751958+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"71770acab1fa63f46c026966890b5231","type":"vertebrate","value":"rabbit","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[1.0,0.52156866,0.96862745],"positions":[{"topLeft":{"x":64.69628,"y":537.62},"width":13.043676,"height":34.858643,"page":1}],"sectionNumber":4,"textBefore":null,"textAfter":null,"comments":[],"startOffset":41,"endOffset":47,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:39:04.328602+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":true,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Dominique Eiflaender	28e437a037	Pull request #464 : RSS-86: Added new rule function redactLineAfterAcrossColumns Merge in RED/redaction-service from RSS-86 to master * commit 'a5f27cfa4cf81bfe1cb29016d18d3fc40e77624d': RSS-86: Added new rule function redactLineAfterAcrossColumns	2022-09-08 13:47:45 +02:00
deiflaender	a5f27cfa4c	RSS-86: Added new rule function redactLineAfterAcrossColumns	2022-09-08 13:05:13 +02:00
Philipp Schramm	34be42cd45	Pull request #463 : RSS-2: Section: added method redactSectionText and extended redactBetween(Not) with excludeHeadline flag Merge in RED/redaction-service from RSS-2 to master * commit 'f1b5d605ccbe4984943e7452dd701004f64f7525': RSS-2: Section: added method redactSectionText and extended redactBetween(Not) with excludeHeadline flag	2022-09-08 12:17:20 +02:00
Philipp Schramm	f1b5d605cc	RSS-2: Section: added method redactSectionText and extended redactBetween(Not) with excludeHeadline flag	2022-09-08 12:10:07 +02:00
Dominique Eiflaender	c3dadd6906	Pull request #462 : RSS-42: Made redactBetween rule more generic Merge in RED/redaction-service from RSS-42 to master * commit 'c29c5eef0d05a86a18fca8978a42ffff3c42ea16': RSS-42: Made redactBetween rule more generic	2022-09-07 14:37:26 +02:00
deiflaender	c29c5eef0d	RSS-42: Made redactBetween rule more generic	2022-09-07 14:31:08 +02:00
Philipp Schramm	9867cd6848	Pull request #461 : RED-5002: Fixed return http error code for testing rules Merge in RED/redaction-service from red-5002 to master * commit '64bd25a9000286c2c30400f1da02cc2ff54ea946': RED-5002: Fixed return http error code for testing rules	2022-08-29 09:43:46 +02:00
Philipp Schramm	64bd25a900	RED-5002: Fixed return http error code for testing rules	2022-08-29 09:38:13 +02:00
Ali Oezyetimoglu	9b9b0ab271	Pull request #459 : RED-4178: Remove special behavior for tables with 2 columns Merge in RED/redaction-service from RED-4178-rs1 to master * commit '29451db72d7cf0b22acb27fd86d3198dab2534ca': RED-4178: Remove special behavior for tables with 2 columns	2022-08-26 16:50:26 +02:00
Ali Oezyetimoglu	da21d7da4b	Pull request #460 : RED-5032: Upgrade services to new base image Merge in RED/redaction-service from RED-5032-rs1 to master * commit 'b6471f904c14e02a7f42cab0aa1dc07bb1aa503a': RED-5032: Upgrade services to new base image	2022-08-26 16:20:58 +02:00
Ali Oezyetimoglu	b6471f904c	RED-5032: Upgrade services to new base image	2022-08-25 17:01:51 +02:00
Ali Oezyetimoglu	29451db72d	RED-4178: Remove special behavior for tables with 2 columns	2022-08-25 11:13:14 +02:00
Dominique Eiflaender	87b76cdae0	Pull request #457 : RED-5022: Add figure detection values the same way as normal images Merge in RED/redaction-service from RED-5022 to master * commit '85cad66ade28fd482beee57b4b01951ec2b93dfd': RED-5022: Add figure detection values the same way as normal images	2022-08-19 14:38:27 +02:00
deiflaender	85cad66ade	RED-5022: Add figure detection values the same way as normal images	2022-08-19 13:50:57 +02:00
Viktor Seifert	1cc93d3a57	Pull request #456 : RED-4824: Added annotations to correctly handle json deserialization Merge in RED/redaction-service from RED-5010 to master * commit 'a1ef711bc32c365b2f276df5de7f3578f82f567f': RED-4824: Corrected annotation for json deserialization so that is unambiguously detected as a delegating creator method RED-4824: Added annotations to correctly handle json deserialization	2022-08-18 09:32:27 +02:00
Viktor Seifert	a1ef711bc3	RED-4824: Corrected annotation for json deserialization so that is unambiguously detected as a delegating creator method	2022-08-17 19:12:28 +02:00
Viktor Seifert	b6a95244d8	RED-4824: Added annotations to correctly handle json deserialization	2022-08-17 18:46:13 +02:00
Ali Oezyetimoglu	4c06ab958c	Pull request #455 : RED-4890 Merge in RED/redaction-service from RED-4890 to master * commit 'f5fb8f7c074b3e7b38c6cfdd28ba4c8cc1edfeff': RED-4890: Refactored redaction log tests to show failed files RED-4890: Refactored RulesTests, will fail individually	2022-08-16 11:16:09 +02:00
Ali Oezyetimoglu	f5fb8f7c07	RED-4890: Refactored redaction log tests to show failed files	2022-08-16 10:56:00 +02:00
Ali Oezyetimoglu	e962992b79	Pull request #454 : RED-4891: Rectangle redactions not listed in DELTA view Merge in RED/redaction-service from RED-4891-rs1 to master * commit 'a9ae01ab32f08d0dadb1b12b0cc4db8070071b42': RED-4891: Rectangle redactions not listed in DELTA view	2022-08-16 09:32:29 +02:00
Ali Oezyetimoglu	a9ae01ab32	RED-4891: Rectangle redactions not listed in DELTA view	2022-08-15 13:49:03 +02:00
Dominique Eiflaender	e77180f6ac	Pull request #453 : RED-3974: Regenerated redactionLog files Merge in RED/redaction-service from RED-3974 to master * commit '269716125c19f8a4029a39566332d20d5cc53a91': RED-3974: Regenerated redactionLog files	2022-08-12 12:29:02 +02:00
deiflaender	269716125c	RED-3974: Regenerated redactionLog files	2022-08-12 12:09:46 +02:00
Dominique Eiflaender	dc21e14921	Pull request #452 : RED-3974: Do not add header to header row Merge in RED/redaction-service from RED-3974 to master * commit '6bc5a6a1351f39bd1815b4fa288faf7bce82b1eb': RED-3974: Do not add header to header row	2022-08-12 11:38:58 +02:00
deiflaender	6bc5a6a135	RED-3974: Do not add header to header row	2022-08-12 11:29:12 +02:00
Dominique Eiflaender	4f66f5acf7	Pull request #451 : RED-3974: Fixed special case table header with diffent size of columns in rows Merge in RED/redaction-service from RED-3974 to master * commit 'a9ff46a3fd3363b47799e8dfe5dc99575c36a33b': RED-3974: Fixed special case table header with diffent size of columns in rows	2022-08-11 17:12:07 +02:00
deiflaender	a9ff46a3fd	RED-3974: Fixed special case table header with diffent size of columns in rows	2022-08-11 17:09:25 +02:00
Dominique Eiflaender	07aaa9722a	Pull request #450 : RED-3974: Use first row as header if header detection does not find a header Merge in RED/redaction-service from RED-3974 to master * commit 'cba81ce061df867936314dcbdbc565248c8db006': RED-3974: Refactored processTablePerRow RED-3974: Use first row as header if header detection does not find a header	2022-08-11 14:28:33 +02:00
deiflaender	cba81ce061	RED-3974: Refactored processTablePerRow	2022-08-11 14:18:36 +02:00
deiflaender	f84a366328	RED-3974: Use first row as header if header detection does not find a header	2022-08-11 11:54:25 +02:00
Viktor Seifert	85c44374d9	Pull request #449 : RED-4824 Merge in RED/redaction-service from RED-4824 to master * commit '1d39c150c7e9084ae6a3833149ad66a4f31da774': RED-4824: Corrected code style by removing parameter assignment RED-4824: Replaced type of the text dir with an enum that only accepts values that can be handled by our code RED-4824: Recreated log files to include changes in rectangle calculation RED-4824: Removed obsolete logging statement RED-4824: Finalized affine transformation by correcting the center of the rotation according to the text direction RED-4824: WIP: Transformed rectangle calculation from a if-else chain to an affine transformation RED-4824: Added test with a simple pdf that contains all combinations for rotation and text direction RED-4824: Cleaned up code to make method more readable RED-4824: Created base test for text-position rectangle creation	2022-08-11 11:09:51 +02:00
Viktor Seifert	1d39c150c7	RED-4824: Corrected code style by removing parameter assignment	2022-08-10 15:49:27 +02:00
Viktor Seifert	b2f1201d92	RED-4824: Replaced type of the text dir with an enum that only accepts values that can be handled by our code	2022-08-10 15:40:20 +02:00
Viktor Seifert	b592fee500	RED-4824: Recreated log files to include changes in rectangle calculation	2022-08-09 18:58:44 +02:00
Viktor Seifert	4f36b8b43e	RED-4824: Removed obsolete logging statement	2022-08-09 17:12:51 +02:00
Viktor Seifert	c7a789ada6	Merge branch 'master' into RED-4824	2022-08-09 16:08:19 +02:00
Viktor Seifert	6de3a3b043	RED-4824: Finalized affine transformation by correcting the center of the rotation according to the text direction	2022-08-09 15:12:10 +02:00
Viktor Seifert	43ff331a42	RED-4824: WIP: Transformed rectangle calculation from a if-else chain to an affine transformation	2022-08-08 19:31:55 +02:00
Timo Bejan	3013dc95f6	Pull request #448 : RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive Merge in RED/redaction-service from RED-4835 to master * commit '6e46cad2c5466136c7191042e6834eb074fc5911': RED-4835 - cleanup RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive	2022-08-08 16:39:49 +02:00
Viktor Seifert	435f75996f	RED-4824: Added test with a simple pdf that contains all combinations for rotation and text direction	2022-08-08 15:16:38 +02:00
Viktor Seifert	ef04b7168a	RED-4824: Cleaned up code to make method more readable	2022-08-08 14:05:25 +02:00
Timo Bejan	6e46cad2c5	RED-4835 - cleanup	2022-08-08 13:31:22 +03:00
Timo Bejan	c41e230f85	RED-4835 - entity position not calculated correctly for duplicates where 1 is marked as false positive	2022-08-08 13:30:18 +03:00
Viktor Seifert	cadbc3c7a4	RED-4824: Created base test for text-position rectangle creation	2022-08-05 16:56:26 +02:00
Philipp Schramm	f4655c4050	RED-4890: Refactored RulesTests, will fail individually	2022-08-04 13:18:05 +02:00
Ali Oezyetimoglu	16ea8364df	Pull request #447 : RED-4867: Sonar issues: RedactionService Merge in RED/redaction-service from RED-4867-rs1 to master * commit '3ec1e20e417e65907448930cf7f8eb9713874fb9': RED-4867: Sonar issues: RedactionService	2022-08-01 16:00:29 +02:00
Ali Oezyetimoglu	3ec1e20e41	RED-4867: Sonar issues: RedactionService	2022-08-01 15:55:22 +02:00
Dominique Eiflaender	ce1f8e9117	Pull request #434 : RED-4753: Reduced size of TEXT File Merge in RED/redaction-service from RED-4753 to master * commit '55bebe8541cfded8a456b63dbd8d8b7e79bab33f': RED-4753: Reduced size of TEXT File a bit more RED-4753: Reduced size of TEXT File	2022-08-01 11:04:25 +02:00
Kresnadi Budisantoso	b7e97890be	Pull request #441 : RED-4829 Fix wrong annotation Merge in RED/redaction-service from kbudisantoso/Sectionjava-1659012500262 to master * commit '47de526b4bd451d302317647cb05ac685e069622': RED-4829 Fix wrong annotation	2022-08-01 09:07:11 +02:00
Dominique Eiflaender	ec49945682	Pull request #446 : RED-4843: Disable jsondsl per default Merge in RED/redaction-service from RED-4843 to master * commit 'b64222938c64eeb006d62807a705918ffd2c585b': RED-4843: Disable jsondsl per default	2022-07-29 12:47:38 +02:00
deiflaender	b64222938c	RED-4843: Disable jsondsl per default	2022-07-29 12:43:35 +02:00
Timo Bejan	e4823e20d4	Pull request #444 : RED-4799 - remove FP entities from redactionlog Merge in RED/redaction-service from RED-4799-3 to master * commit '53428ec65296b9fac2c206e7bd89916c933fb5c8': RED-4799 - remove FP entities from redactionlog	2022-07-29 10:53:51 +02:00
Timo Bejan	53428ec652	RED-4799 - remove FP entities from redactionlog	2022-07-29 11:50:38 +03:00
Dominique Eiflaender	ca270fb7de	Pull request #443 : RED-4843: Upgraded storage commons Merge in RED/redaction-service from RED-4843 to master * commit '06fe0dbd97cdf2011be17df07f8c409a221530a4': RED-4843: Upgraded storage commons	2022-07-29 10:41:24 +02:00
deiflaender	06fe0dbd97	RED-4843: Upgraded storage commons	2022-07-29 10:27:18 +02:00
Kresnadi Budisantoso	47de526b4b	RED-4829 Fix wrong annotation	2022-07-28 14:48:39 +02:00
Timo Bejan	1d8e86e4f6	Pull request #439 : RED-4799 Merge in RED/redaction-service from RED-4799-fix to master * commit '2b7972315f655bd296a03cae8cc5e4846d47bbce': RED-4799	2022-07-27 11:58:11 +02:00
Timo Bejan	2b7972315f	RED-4799	2022-07-27 12:54:51 +03:00
Philipp Schramm	0af853dbfc	Pull request #436 : RED-4510: Added generated RedactionLog Json files and optimizations for RulesTest.java for Bamboo Merge in RED/redaction-service from RED-4510-Rework to master * commit '0544a42117a742de70325cd1a84e57d6829aaa25': RED-4510: Added generated RedactionLog Json files and optimizations for RulesTest.java for Bamboo	2022-07-26 17:08:59 +02:00
Philipp Schramm	0544a42117	RED-4510: Added generated RedactionLog Json files and optimizations for RulesTest.java for Bamboo	2022-07-26 17:06:04 +02:00
deiflaender	55bebe8541	RED-4753: Reduced size of TEXT File a bit more	2022-07-26 11:55:25 +02:00
Timo Bejan	8bdfd6747d	Pull request #435 : RED-4686 Merge in RED/redaction-service from tbejan/pomxml-1658825710723 to master * commit '80c0c25c069995790d6bd043657f50dc8de0a451': RED-4686	2022-07-26 11:01:22 +02:00
Timo Bejan	80c0c25c06	RED-4686	2022-07-26 10:55:17 +02:00
deiflaender	1cc66a3092	RED-4753: Reduced size of TEXT File	2022-07-26 09:45:46 +02:00
Timo Bejan	17baf9c0eb	Pull request #433 : RED-4686 - updated cyclic deps Merge in RED/redaction-service from RED-4686 to master * commit '090a6196000dfcd7a447c54224fba8eda4605380': RED-4686 - updated cyclic deps	2022-07-26 09:25:58 +02:00
Timo Bejan	090a619600	RED-4686 - updated cyclic deps	2022-07-26 10:22:50 +03:00
				`@@ -0,0 +1 @@`
				{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"12ac3d26ce82a77e469c64d56081662d","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Table in: COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.8,"y":651.16003},"width":54.039078,"height":-11.520004,"page":4}],"sectionNumber":3,"textBefore":"Report: Document F/02, ","textAfter":". (2014). Cyprodinil","comments":[],"startOffset":23,"endOffset":35,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:36:30.044122+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dd614353319e7c5a8cf8b3346dc12fac","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":306.84,"y":679.12},"width":59.71997,"height":-12.0,"page":4}],"sectionNumber":5,"textBefore":"F report (","textAfter":"., 2014).","comments":[],"startOffset":247,"endOffset":259,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:36:30.044134+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
				`@@ -0,0 +1 @@`
				`{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}`