RED-4510: Generated Redaction Log and Generating will not run with profile

redaction-service-v1/pom.xml

@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>1.17.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.254.0</persistence-service.version>
+        <persistence-service.version>1.240.0</persistence-service.version>
     </properties>
 
     <dependencies>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/FloatFrequencyCounter.java

@@ -35,7 +35,9 @@ public class FloatFrequencyCounter {
     public Float getMostPopular() {
         Map.Entry<Float, Integer> mostPopular = null;
         for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
+            if (mostPopular == null) {
+                mostPopular = entry;
+            } else if (entry.getValue() >= mostPopular.getValue()) {
                 mostPopular = entry;
             }
         }
@@ -59,7 +61,9 @@ public class FloatFrequencyCounter {
     public Float getHighest() {
         Float highest = null;
         for (Float value : countPerValue.keySet()) {
-            if (highest == null || value > highest) {
+            if (highest == null) {
+                highest = value;
+            } else if (value > highest) {
                 highest = value;
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/StringFrequencyCounter.java

@@ -37,7 +37,9 @@ public class StringFrequencyCounter {
 
         Map.Entry<String, Integer> mostPopular = null;
         for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
+            if (mostPopular == null) {
+                mostPopular = entry;
+            } else if (entry.getValue() > mostPopular.getValue()) {
                 mostPopular = entry;
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -23,30 +23,16 @@ public class TextBlock extends AbstractTextContainer {
 
     @Builder.Default
     private List<TextPositionSequence> sequences = new ArrayList<>();
-
-    @JsonIgnore
     private int rotation;
 
-    @JsonIgnore
     private String mostPopularWordFont;
-
-    @JsonIgnore
     private String mostPopularWordStyle;
-
-    @JsonIgnore
     private float mostPopularWordFontSize;
-
-    @JsonIgnore
     private float mostPopularWordHeight;
-
-    @JsonIgnore
     private float mostPopularWordSpaceWidth;
 
-    @JsonIgnore
     private float highestFontSize;
 
-
-    @JsonIgnore
     private String classification;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,33 +1,29 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RestController;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.*;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.stream.Collectors;
 
 @Slf4j
 @RestController

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java

@@ -5,6 +5,7 @@ import org.springframework.beans.BeanUtils;
 
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
 import lombok.AllArgsConstructor;
@@ -21,20 +22,22 @@ import lombok.SneakyThrows;
 public class RedTextPosition {
 
     private String textMatrix;
-    private float[] position;
-
-    @JsonIgnore
     private int rotation;
-
-    @JsonIgnore
+    private float y;
     private float pageHeight;
-
-    @JsonIgnore
     private float pageWidth;
-
     private String unicode;
 
-    @JsonIgnore
+    @JsonAlias("xdirAdj")
+    @JsonAttribute(alternativeNames = {"xdirAdj"})
+    private float XDirAdj;
+
+    @JsonAlias("ydirAdj")
+    @JsonAttribute(alternativeNames = {"ydirAdj"})
+    private float YDirAdj;
+    private float width;
+    private float heightDir;
+    private float widthDirAdj;
     private float dir;
 
     // not used in reanalysis
@@ -63,37 +66,18 @@ public class RedTextPosition {
 
         pos.setTextMatrix(textPosition.getTextMatrix().toString());
 
-        var position = new float[4];
-
-        position[0] = textPosition.getXDirAdj();
-        position[1] = textPosition.getYDirAdj();
-        position[2] = textPosition.getWidthDirAdj();
-        position[3] = textPosition.getHeightDir();
-
-        pos.setPosition(position);
         return pos;
     }
 
 
-    @JsonIgnore
-    public float getXDirAdj(){
-        return position[0];
-    }
+    @JsonAlias("xdirAdj")
+    @JsonAttribute(alternativeNames = {"xdirAdj"})
+    public void setXDirAdj(float XDirAdj) {this.XDirAdj = XDirAdj;}
 
-    @JsonIgnore
-    public float getYDirAdj(){
-        return position[1];
-    }
 
-    @JsonIgnore
-    public float getWidthDirAdj(){
-        return position[2];
-    }
-
-    @JsonIgnore
-    public float getHeightDir(){
-        return position[3];
-    }
+    @JsonAlias("ydirAdj")
+    @JsonAttribute(alternativeNames = {"ydirAdj"})
+    public void setYDirAdj(float YDirAdj) {this.YDirAdj = YDirAdj;}
 
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -31,10 +31,8 @@ public class TextPositionSequence implements CharSequence {
     private int page;
     private List<RedTextPosition> textPositions = new ArrayList<>();
 
-    private float dir;
-    private int rotation;
-    private float pageHeight;
-    private float pageWidth;
+    private float x1;
+    private float x2;
 
 
     public TextPositionSequence(int page) {
@@ -43,7 +41,7 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
+    public static TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
 
         var textPositionSequence = new TextPositionSequence();
         textPositionSequence.textPositions = textPositions;
@@ -57,10 +55,6 @@ public class TextPositionSequence implements CharSequence {
 
         this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
         this.page = page;
-        this.dir = textPositions.get(0).getDir();
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
     }
 
 
@@ -91,15 +85,7 @@ public class TextPositionSequence implements CharSequence {
     @Override
     public TextPositionSequence subSequence(int start, int end) {
 
-        var textPositionSequence = new TextPositionSequence();
-        textPositionSequence.textPositions = textPositions.subList(start, end);
-        textPositionSequence.page = page;
-        textPositionSequence.dir = dir;
-        textPositionSequence.rotation = rotation;
-        textPositionSequence.pageHeight = pageHeight;
-        textPositionSequence.pageWidth = pageWidth;
-
-        return textPositionSequence;
+        return fromData(textPositions.subList(start, end), page);
     }
 
 
@@ -120,26 +106,15 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
+    public void add(RedTextPosition textPosition) {
 
         this.textPositions.add(textPosition);
-        this.page = textPositionSequence.getPage();
-        this.dir = textPositionSequence.getDir();
-        this.rotation = textPositionSequence.getRotation();
-        this.pageHeight = textPositionSequence.getPageHeight();
-        this.pageWidth = textPositionSequence.getPageWidth();
     }
 
 
     public void add(TextPosition textPosition) {
 
         this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
-
-        this.dir = textPositions.get(0).getDir();
-        this.rotation = textPositions.get(0).getRotation();
-        this.pageHeight = textPositions.get(0).getPageHeight();
-        this.pageWidth = textPositions.get(0).getPageWidth();
-
     }
 
 
@@ -147,7 +122,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getX1() {
 
-        if (rotation == 90) {
+        if (textPositions.get(0).getRotation() == 90) {
             return textPositions.get(0).getYDirAdj() - getTextHeight();
         } else {
             return textPositions.get(0).getXDirAdj();
@@ -159,11 +134,10 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getX2() {
 
-        if (rotation == 90) {
+        if (textPositions.get(0).getRotation() == 90) {
             return textPositions.get(0).getYDirAdj();
         } else {
-            return textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
+            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
         }
     }
 
@@ -172,7 +146,15 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getRotationAdjustedY() {
 
-        return textPositions.get(0).getYDirAdj();
+        return textPositions.get(0).getY();
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getRotationAdjustedX() {
+
+        return textPositions.get(0).getXDirAdj();
     }
 
 
@@ -180,10 +162,10 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getY1() {
 
-        if (rotation == 90) {
+        if (textPositions.get(0).getRotation() == 90) {
             return textPositions.get(0).getXDirAdj();
         } else {
-            return pageHeight - textPositions.get(0).getYDirAdj();
+            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj();
         }
     }
 
@@ -192,10 +174,10 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getY2() {
 
-        if (rotation == 90) {
+        if (textPositions.get(0).getRotation() == 90) {
             return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - 2;
         } else {
-            return pageHeight - textPositions.get(0).getYDirAdj() + getTextHeight();
+            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() + getTextHeight();
         }
     }
 
@@ -267,11 +249,19 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public int getRotation() {
+
+        return textPositions.get(0).getRotation();
+    }
+
+
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public Rectangle getRectangle() {
 
-        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), rotation, dir);
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, textPositions.get(0).getRotation(), textPositions.get(0).getDir());
 
         float height = getTextHeight();
 
@@ -280,132 +270,113 @@ public class TextPositionSequence implements CharSequence {
         float posYInit;
         float posYEnd;
 
-        if (rotation == 0 && dir == 90f) {
+        if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 90f) {
 
             posYInit = getX1();
-            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
-                    .getWidthDirAdj() - 3;
+            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
             posXInit = textPositions.get(0).getYDirAdj() + 2;
             posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
 
-        } else if (rotation == 0 && dir == 180f) {
+        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 180f) {
 
-            posXInit = pageWidth - getX1() + 1;
-            posXEnd = pageWidth - getX2() + textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
             posYInit = textPositions.get(0).getYDirAdj() - height + 2;
             posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
 
-        } else if (rotation == 0 && dir == 270f) {
+        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) {
 
-            posYInit = pageHeight - getX1();
-            posYEnd = pageHeight - getX2() - textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
-            posXInit = pageWidth - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = pageWidth - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
+            posYInit = textPositions.get(0).getPageHeight() - getX1();
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
 
-        } else if (rotation == 90 && dir == 0.0f) {
+        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 0.0f) {
 
-            posXInit = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
+            posXInit = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
             posXEnd = textPositions.get(0).getXDirAdj();
-            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
+            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
 
-        } else if (rotation == 90 && dir == 90.0f) {
+        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 90.0f) {
             posXEnd = textPositions.get(0).getYDirAdj() + 2;
             posYInit = getY1();
             posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
 
-        } else if (rotation == 90 && dir == 180.0f) {
+        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 180.0f) {
 
-            posXInit = pageWidth - textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() - 4;
-            posXEnd = pageWidth - textPositions.get(0).getXDirAdj();
-            posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1)
-                    .getHeightDir();
-            posYEnd = textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getXDirAdj() - 4;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getXDirAdj();
+            posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1).getHeightDir();
+            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();
 
-        } else if (rotation == 90 && dir == 270.0f) {
+        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 270.0f) {
 
-            posXInit = pageWidth - getX1();
-            posXEnd = pageWidth - textPositions.get(0).getYDirAdj() - 2;
-            posYInit = pageHeight - getY1();
-            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() - height - 4;
+            posXInit = textPositions.get(0).getPageWidth() - getX1();
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
+            posYInit = textPositions.get(0).getPageHeight() - getY1();
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getXDirAdj() - height - 4;
 
-        } else if (rotation == 180 && dir == 0f) {
+        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 0f) {
 
-            posXEnd = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
-            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
+            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
+            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
 
-        } else if (rotation == 180 && dir == 90f) {
+        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 90f) {
 
             posYInit = getX1();
             posYEnd = getX2() - 3;
             posXInit = textPositions.get(0).getYDirAdj() + 2;
             posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
 
-        } else if (rotation == 180 && dir == 180f) {
+        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
 
-            posXInit = pageWidth - getX1() + 1;
-            posXEnd = pageWidth - getX2() + textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
             posYInit = textPositions.get(0).getYDirAdj() - height + 2;
             posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
 
-        } else if (rotation == 180 && dir == 270.0f) {
+        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 270.0f) {
 
-            posYInit = pageHeight - getX1();
-            posYEnd = pageHeight - getX2() - textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
-            posXInit = pageWidth - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = pageWidth - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
+            posYInit = textPositions.get(0).getPageHeight() - getX1();
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
 
-        } else if (rotation == 270 && dir == 0.0f) {
+        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) {
 
-            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
+            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
             posYEnd = posYInit + 1;
             posXInit = textPositions.get(0).getXDirAdj();
-            posXEnd = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
+            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
 
-        } else if (rotation == 270 && dir == 90.0f) {
+        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 90.0f) {
 
             posYInit = getX1();
             posYEnd = getX2() - height;
             posXInit = textPositions.get(0).getYDirAdj() + 2;
             posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
 
-        } else if (rotation == 270 && dir == 180.0f) {
+        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 180.0f) {
 
-            posXInit = pageWidth - getX1() + 1;
-            posXEnd = pageWidth - getX2() - 4;
+            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() - 4;
             posYInit = textPositions.get(0).getYDirAdj() - height + 2;
             posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
 
-        } else if (rotation == 270 && dir == 270.0f) {
+        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 270.0f) {
 
-            posYInit = pageHeight - getX1();
-            posYEnd = pageHeight - getX2() - height;
-            posXInit = pageWidth - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = pageWidth - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
+            posYInit = textPositions.get(0).getPageHeight() - getX1();
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - height;
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
 
         } else {
             // page rotation = 0 and text direction = 0
-            posXEnd = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
-            posYInit = pageHeight - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = pageHeight - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
+            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
+            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
         }
 
         var rectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,14 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import lombok.*;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
 @Data
 @Builder
 @NoArgsConstructor
@@ -19,7 +24,6 @@ public class Entity implements ReasonHolder {
     private String word;
     private String type;
     private boolean redaction;
-    private boolean falsePositive;
     private String redactionReason;
     private String legalBasis;
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -100,7 +100,7 @@ public class SearchableText {
                                     .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                             .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
                             .charAt(j, caseInsensitive) != ' ') {
-                        partMatch.add(searchSpace.get(i),searchSpace.get(i).textPositionAt(j));
+                        partMatch.add(searchSpace.get(i).textPositionAt(j));
                         if (!(j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
                                 .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                             counter++;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1,5 +1,23 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
@@ -10,19 +28,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+
 import lombok.Builder;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 @Data
 @Slf4j
@@ -69,7 +78,7 @@ public class Section {
 
 
     @SuppressWarnings("unused")
-    @ThenAction
+    @WhenCondition
     public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
 
         Set<Entity> entitiesOfType = nerEntities.stream()
@@ -78,7 +87,6 @@ public class Section {
         List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
         Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
         EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
 
         Set<Entity> finalResult = new HashSet<>();
 
@@ -104,7 +112,7 @@ public class Section {
 
 
     @SuppressWarnings("unused")
-    @ThenAction
+    @WhenCondition
     public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType,
                                @Argument(ArgumentType.TYPE) String combineTypes,
                                @Argument(ArgumentType.INTEGER) int maxDistanceBetween,
@@ -878,11 +886,13 @@ public class Section {
             }
         });
 
-        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
+        return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
     }
 
 
+
+
+
     private void redact(String type, int ruleNumber, String reason, String legalBasis, boolean redaction) {
 
         entities.forEach(entity -> {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,5 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
@@ -9,21 +22,24 @@ import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 @Slf4j
 @Service
@@ -35,6 +51,7 @@ public class EntityRedactionService {
     private final SurroundingWordsService surroundingWordsService;
 
 
+
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer,
                                      AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
 
@@ -245,8 +262,7 @@ public class EntityRedactionService {
             nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
         }
 
-        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
+        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions), nerFound);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -5,6 +5,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
@@ -12,6 +13,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@@ -112,7 +114,10 @@ public class RedactionLogCreatorService {
                 redactionLogEntry.setId(entityPositionSequence.getId());
 
                 if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
-                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences());
+                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
+                            .stream()
+                            .flatMap(seq -> seq.getTextPositions().stream())
+                            .collect(Collectors.toList()), page);
 
                     redactionLogEntry.getPositions().addAll(rectanglesPerLine);
 
@@ -129,43 +134,38 @@ public class RedactionLogCreatorService {
     }
 
 
-    private List<Rectangle> getRectanglesPerLine(List<TextPositionSequence> textPositionSequences) {
+    private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {
 
         List<Rectangle> rectangles = new ArrayList<>();
+        if (textPositions.size() == 1) {
+            rectangles.add(TextPositionSequence.fromData(textPositions, page).getRectangle());
+        } else {
+            float x = textPositions.get(0).getXDirAdj();
+            float y = textPositions.get(0).getYDirAdj();
+            float width = textPositions.get(0).getWidthDirAdj();
+            float height = textPositions.get(0).getHeightDir();
+            int startIndex = 0;
 
-        if (textPositionSequences.size() == 1) {
-            rectangles.add(textPositionSequences.get(0).getRectangle());
-            return rectangles;
-        }
+            for (int i = 1; i < textPositions.size(); i++) {
+                float xDirAdj = textPositions.get(i).getXDirAdj();
+                float yDirAdj = textPositions.get(i).getYDirAdj();
+                float widthDir = textPositions.get(i).getWidthDirAdj();
+                float heightDir = textPositions.get(i).getHeightDir();
 
-        TextPositionSequence combinedSequence = new TextPositionSequence();
-        for (int i = 0; i < textPositionSequences.size(); i++) {
-
-            if(combinedSequence.getTextPositions().isEmpty()){
-                combinedSequence = textPositionSequences.get(i);
-                continue;
+                if (!(isCharInSameLine(y, yDirAdj, height, heightDir) && isCharClose(x, xDirAdj, width))) {
+                    rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, i), page)
+                            .getRectangle());
+                    y = yDirAdj;
+                    width = widthDir;
+                    height = heightDir;
+                    startIndex = i;
+                }
+                x = xDirAdj;
             }
-
-            float lastSeqX = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getXDirAdj();
-            float lastSeqY = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getYDirAdj();
-            float lastSeqWidth = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getWidthDirAdj();
-            float lastSeqHeight = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getHeightDir();
-
-            float currentSeqX = textPositionSequences.get(i).getTextPositions().get(0).getXDirAdj();
-            float currentSeqY = textPositionSequences.get(i).getTextPositions().get(0).getYDirAdj();
-            float currentSeqHeight = textPositionSequences.get(i).getTextPositions().get(0).getHeightDir();
-
-            if (isCharInSameLine(lastSeqY, currentSeqY, lastSeqHeight, currentSeqHeight) && isCharClose(lastSeqX, currentSeqX, lastSeqWidth)) {
-                combinedSequence.getTextPositions().addAll(textPositionSequences.get(i).getTextPositions());
-            } else {
-                rectangles.add(combinedSequence.getRectangle());
-                combinedSequence = textPositionSequences.get(i);
+            if (startIndex != textPositions.size()) {
+                rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page)
+                        .getRectangle());
             }
-
-        }
-
-        if(!combinedSequence.getTextPositions().isEmpty()) {
-            rectangles.add(combinedSequence.getRectangle());
         }
 
         return rectangles;
@@ -207,8 +207,7 @@ public class RedactionLogCreatorService {
                 .redacted(entity.isRedaction())
                 .isHint(isHint(entity.getType(), dossierTemplateId))
                 .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType()
-                        .equals(EntityType.FALSE_RECOMMENDATION))
+                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
                 .section(entity.getHeadline())
                 .sectionNumber(entity.getSectionNumber())
                 .matchedRule(entity.getMatchedRule())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,15 +1,28 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @UtilityClass
 @SuppressWarnings("PMD")
@@ -26,10 +39,12 @@ public class EntitySearchUtils {
 
         if (details.getEntityType() == EntityType.RECOMMENDATION) {
             Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
-            markFalsePositives(found, falseRecommendations);
+            removeFalsePositives(found, falseRecommendations);
+            found.addAll(falseRecommendations);
         } else {
             Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
-            markFalsePositives(found, falsePositives);
+            removeFalsePositives(found, falsePositives);
+            found.addAll(falsePositives);
         }
 
         return found;
@@ -78,7 +93,7 @@ public class EntitySearchUtils {
             }
         }
 
-        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()) {
+        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()){
             applyResizeRedactions(entities, manualRedactions);
         }
 
@@ -116,11 +131,7 @@ public class EntitySearchUtils {
     }
 
 
-    public void markFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
-
-        if (entities.size() == 2) {
-            log.info("asd");
-        }
+    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
 
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : falsePositives) {
@@ -131,12 +142,8 @@ public class EntitySearchUtils {
                 }
             }
         }
-        wordsToRemove.forEach(e -> e.setFalsePositive(true));
-        for (var entity : entities) {
-            if (falsePositives.contains(entity)) {
-                entity.setFalsePositive(true);
-            }
-        }
+        wordsToRemove.forEach(entities::remove);
+        entities.removeAll(falsePositives);
     }
 
 
@@ -145,22 +152,13 @@ public class EntitySearchUtils {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
-//                // skip cross-type false positives
-//                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
-//                    continue;
-//                }
-//                // skip cross-type false reccommendations
-//                if(word.getEntityType() == EntityType.FALSE_RECOMMENDATION && inner.getEntityType() == EntityType.RECOMMENDATION && !inner.getType().equals(word.getType())){
-//                    continue;
-//                }
-
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                         wordsToRemove.add(word);
-                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
                             inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
-                        if (inner.isResized()) {
+                        if(inner.isResized()){
                             wordsToRemove.add(word);
                         } else {
                             wordsToRemove.add(inner);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -1,9 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.storage;
 
-import java.io.InputStream;
-
-import org.springframework.stereotype.Service;
-
+import com.dslplatform.json.DslJson;
+import com.dslplatform.json.runtime.Settings;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
@@ -13,12 +12,16 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import io.micrometer.core.annotation.Timed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
 
 @Slf4j
 @Service

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java

@@ -13,20 +13,13 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public abstract class AbstractTextContainer {
 
-    @JsonIgnore
     protected float minX;
-    @JsonIgnore
     protected float maxX;
-    @JsonIgnore
     protected float minY;
-    @JsonIgnore
     protected float maxY;
-    @JsonIgnore
     protected String classification;
-    @JsonIgnore
     protected int page;
 
-    @JsonIgnore
     private Orientation orientation = Orientation.NONE;
 
     public abstract String getText();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -1,5 +1,14 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
+import org.springframework.core.io.InputStreamResource;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -10,16 +19,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
 public class FileSystemBackedStorageService implements StorageService {
 
     private final Map<String, File> dataMap = new HashMap<>();
@@ -63,8 +62,11 @@ public class FileSystemBackedStorageService implements StorageService {
     }
 
     private ObjectMapper getMapper() {
-
-        return ObjectMapperFactory.create();
+        var objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.findAndRegisterModules();
+        return objectMapper;
     }
 
     @Override

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -113,13 +113,12 @@ public class RulesTest {
     private static final String IMPORTED_REDACTION = "imported_redaction";
     private static final String PII = "PII";
     private static final String RESOURCES_PATH = "src/test/resources/";
+
     private static final String REDACTION_LOG_PATH = "RedactionLog/";
     private static final String REDACTION_LOG_RESOURCES_PATH = RESOURCES_PATH + REDACTION_LOG_PATH;
     private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
     private final static String TEST_DOSSIER_ID = "123";
     private String TEST_FILE_ID = "123";
-    private int counter = 0;
-    private int fileSize = 0;
     private final Map<String, List<String>> dictionary = new HashMap<>();
     private final Map<String, List<String>> dossierDictionary = new HashMap<>();
     private final Map<String, List<String>> falsePositive = new HashMap<>();
@@ -178,9 +177,7 @@ public class RulesTest {
                 .hexColor("#ffe187")
                 .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
                 .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
-                .build()));
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
 
         mockDictionaryCalls(null);
         mockDictionaryCalls(0L);
@@ -208,7 +205,6 @@ public class RulesTest {
     public void generateRedactionLogForOneFile() {
 
         String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
-        fileSize = 1;
         generateAndSaveRedactionLog(fileName, true);
     }
 
@@ -219,24 +215,28 @@ public class RulesTest {
      * Please commit generated files, if not the following tests will fail
      */
     @Test
+    @Ignore// TODO PSC REMOVE
     public void generateRedactionLogForAllFiles() {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will generate RedactionLog for " + files.size() + " files.");
-        fileSize = files.size();
         files.forEach(this::generateAndSaveRedactionLog);
     }
 
 
+
+
+
     /**
      * Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
      * If RedactionLog Json does not exist, test will fail.
+     * Test is ignored, because it's for manual tests.
      */
+    @Ignore
     @Test
     public void analyseFileAndCompareRedactionLog() {
 
-        String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf";
-        fileSize = 1;
+        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
         analyseFileAndCompareRedactionLog(fileName);
     }
 
@@ -251,28 +251,35 @@ public class RulesTest {
 
         Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
         System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
-        fileSize = files.size();
+
+        System.out.println("files: " + files);
+
         files.forEach(this::analyseFileAndCompareRedactionLog);
     }
 
 
-    private void generateAndSaveRedactionLog(String fileName) {
-
-        generateAndSaveRedactionLog(fileName, false);
+    @Test// TODO PSC REMOVE
+    @Ignore
+    public void foo() {
+        String fileName = "oo-home/local-working-dir/3080193/RED-REDACTIONSERVICENIGHT0-JOB1/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Compounds/26 A8637C - EU AIR3 - LCP Section 10 - Ecotoxicological studies on the plant protection product - Reference list.pdf";
+        loadSavedRedactionLog(fileName);
     }
 
 
+    private void generateAndSaveRedactionLog(String fileName) {
+        generateAndSaveRedactionLog(fileName, false);
+    }
+
     @SneakyThrows
     private void generateAndSaveRedactionLog(String fileName, boolean force) {
 
-        counter++;
         if (!force && doesRedactionLogExist(fileName)) {
-            System.out.println("(" + counter + "/" + fileSize + ") Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
+            System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
 
         } else {
             generateTestFileId(fileName);
 
-            System.out.println("(" + counter + "/" + fileSize + ") Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+            System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
 
             loadNerForTest();
 
@@ -291,9 +298,8 @@ public class RulesTest {
     @SneakyThrows
     public void analyseFileAndCompareRedactionLog(String fileName) {
 
-        counter++;
         generateTestFileId(fileName);
-        System.out.println("(" + counter + "/" + fileSize + ") Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
+        System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
 
         RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
 
@@ -313,7 +319,7 @@ public class RulesTest {
         assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
         assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
 
-        assertThat(redactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size());
+        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
         assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
 
         for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
@@ -401,7 +407,7 @@ public class RulesTest {
                 assertThat(savedManualChange).isPresent();
             }
 
-            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(new Engine[0]));
+            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
 
             assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
             assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
@@ -454,53 +460,27 @@ public class RulesTest {
     @SneakyThrows
     private RedactionLog loadSavedRedactionLog(String pdfFileName) {
 
+        System.out.println("XXXXXXXXX pdfFileName: " + pdfFileName);
+
         File pdfFile = new File(pdfFileName);
-        String directory = cleanPath(pdfFile.getParentFile().getPath());
-        if (StringUtils.contains(directory, RESOURCES_PATH)) {
-            if (!StringUtils.endsWith(directory, "/")) {
-                directory = directory + "/";
-            }
-            directory = directory + REDACTION_LOG_PATH;
+        String directory = pdfFile.getParentFile().getPath();
+        if (StringUtils.contains(directory, RESOURCES_PATH) || StringUtils.contains(directory, StringUtils.replace(RESOURCES_PATH, "/", "\\"))) {
+            directory = directory + "/" + REDACTION_LOG_PATH;
         } else {
-            if (StringUtils.startsWith(directory, "/")) {
-                directory = StringUtils.substring(directory, 1);
-            }
             directory = REDACTION_LOG_RESOURCES_PATH + directory;
         }
         String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
-
         File file = new File(directory, fileName);
+
+        System.out.println("XXXXXXXXX directory: " + directory);
+        System.out.println("XXXXXXXXX fileName:  " + fileName);
+
         assertThat(file).exists();
 
         return objectMapper.readValue(file, RedactionLog.class);
     }
 
 
-    @SneakyThrows
-    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
-
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
-            for (Path path : stream) {
-                if (path.toFile().isDirectory()) {
-                    getFileNames(fileNames, path);
-                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
-                    String absolutePath = cleanPath(path.toAbsolutePath().toString());
-                    int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length();
-
-                    fileNames.add(StringUtils.substring(absolutePath, pos));
-                }
-            }
-        }
-        return fileNames;
-    }
-
-
-    private String cleanPath(String path) {
-
-        return StringUtils.replace(path, "\\", "/");
-    }
-
-
     @SneakyThrows
     private void loadNerForTest() {
 
@@ -639,6 +619,30 @@ public class RulesTest {
     }
 
 
+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = path.toAbsolutePath().toString();
+                    System.out.println("absolutePath: " + absolutePath);
+                    int pos = StringUtils.indexOf(absolutePath, StringUtils.replace(RESOURCES_PATH, "/", "\\")) + 18;
+
+                    pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + 18;
+
+                    System.out.println("pos: " + pos);
+                    fileNames.add(StringUtils.substring(absolutePath, pos));
+                    System.out.println("Added: " + StringUtils.substring(absolutePath, pos));
+                }
+            }
+        }
+        return fileNames;
+    }
+
+
     private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
 
         return Type.builder()

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -1,15 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.annotate;
 
-import java.awt.Color;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import lombok.RequiredArgsConstructor;
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -23,21 +22,12 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.CellRectangle;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.model.SectionRectangle;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import lombok.RequiredArgsConstructor;
+import java.awt.*;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.*;
+import java.util.stream.Collectors;
 
 @Service
 @RequiredArgsConstructor

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java

@@ -5,11 +5,9 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
 import java.time.OffsetDateTime;
@@ -21,9 +19,6 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
 
     public static final String FILE_NAME = "test-file";
 
-    @Autowired
-    private AnnotationService annotationService;
-
     @Test
     @SneakyThrows
     public void testFile() {
@@ -86,19 +81,7 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
         redactionMessageReceiver.receiveAnalyzeRequest(in, false);
 
 
-//        var redactionLog = redactionStorageService.getRedactionLog("dossierId","fileId");
-//        redactionLog.getRedactionLogEntry().forEach(entry ->{
-//            log.info("Entry {}", entry);
-//        });
-//
-//        var annotated = annotationService.annotate(AnnotateRequest.builder().fileId("fileId").dossierId("dossierId").build());
-//        annotated.getDocument();
-//
-//        File tempFile = File.createTempFile("annotated",".pdf");
-//        IOUtils.write(annotated.getDocument(), new FileOutputStream(tempFile));
-//        log.warn("File saved to: {}",tempFile.getAbsolutePath());
-//        Runtime.getRuntime().exec("open "+tempFile.getAbsolutePath());
+        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -1,21 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.realdata;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.*;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import lombok.SneakyThrows;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -34,14 +30,27 @@ import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.when;
+import lombok.SneakyThrows;
 
 @RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -134,7 +143,6 @@ public class LiveDataIntegrationTest {
             if (found.isPresent()) {
                 var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
                 type.setEntries(getEntries(typeName, type.getTypeId()));
-                type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId()));
 
                 return type;
             } else {
@@ -188,22 +196,7 @@ public class LiveDataIntegrationTest {
 
         Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
         for (var resource : dictionaryResources) {
-            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) {
-
-                List<String> lines = IOUtils.readLines(resource.getInputStream());
-                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
-
-            }
-        }
-        return new ArrayList<>();
-    }
-
-    @SneakyThrows
-    private List<DictionaryEntry> getFalsePositiveEntries(String typeName, String typeId) {
-
-        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
-        for (var resource : dictionaryResources) {
-            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName)) {
 
                 List<String> lines = IOUtils.readLines(resource.getInputStream());
                 return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java

@@ -1,13 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+
 public class RegExPatternTest {
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/performance/data/test-file-2.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/dictionaries/CBI_author_false_positive.txt

@@ -1 +0,0 @@
-Salt), CGA376944 (S-enantiomer