Pull request #464: RSS-86: Added new rule function redactLineAfterAcrossColumns

Merge in RED/redaction-service from RSS-86 to master

* commit 'a5f27cfa4cf81bfe1cb29016d18d3fc40e77624d':
  RSS-86: Added new rule function redactLineAfterAcrossColumns

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -1,10 +1,12 @@
 package buildjob;
 
-import java.time.DayOfWeek;
+import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+
 import java.time.LocalTime;
 
 import com.atlassian.bamboo.specs.api.BambooSpec;
 import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.Variable;
 import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
 import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
 import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
@@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.api.builders.Variable;
+import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
 import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
 import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
-import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
 import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
 import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
-import com.atlassian.bamboo.specs.util.BambooServer;
-import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
-
-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+import com.atlassian.bamboo.specs.util.BambooServer;
 
 /**
  * Plan configuration for Bamboo.
@@ -41,14 +39,15 @@ public class PlanSpec {
 
     private static final String SERVICE_NAME = "redaction-service";
 
-    private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
+    private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
 
     private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
 
+
     /**
      * Run main to publish plan on Bamboo
      */
-    public static void main(final String[] args) throws Exception {
+    public static void main(final String[] args) {
         //By default credentials are read from the '.credentials' file.
         BambooServer bambooServer = new BambooServer("http://localhost:8085");
 
@@ -57,15 +56,21 @@ public class PlanSpec {
         PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
         bambooServer.publish(planPermission);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        bambooServer.publish(nightPlan);
+        PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
+        bambooServer.publish(nightPlanPermission);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         bambooServer.publish(secPlan);
         PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
         bambooServer.publish(secPlanPermission);
     }
 
+
     private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
-        Permissions permission = new Permissions()
-                .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
+
+        Permissions permission = new Permissions().userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
                 .groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
                 .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
                 .loggedInUserPermissions(PermissionType.VIEW)
@@ -73,105 +78,80 @@ public class PlanSpec {
         return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
     }
 
+
     private Project project() {
-        return new Project()
-                .name("RED")
-                .key(new BambooKey("RED"));
+
+        return new Project().name("RED").key(new BambooKey("RED"));
     }
 
+
     public Plan createPlan() {
-        return new Plan(
-                project(),
-                SERVICE_NAME, new BambooKey(SERVICE_KEY))
-                .description("Plan created from (enter repository url of your plan)")
+
+        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
                 .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage")
-                        .jobs(new Job("Default Job",
-                                new BambooKey("JOB1"))
-                                .tasks(
-                                        new ScriptTask()
-                                                .description("Clean")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-                                                        "rm -rf ./*"),
-                                        new VcsCheckoutTask()
-                                                .description("Checkout Default Repository")
-                                                .cleanCheckout(true)
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
-                                        new ScriptTask()
-                                                .description("Build")
-                                                .location(Location.FILE)
-                                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                                .argument(SERVICE_NAME),
-                                        createJUnitParserTask()
-                                                .description("Resultparser")
-                                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                                .enabled(true),
-                                        new InjectVariablesTask()
-                                                .description("Inject git Tag")
-                                                .path("git.tag")
-                                                .namespace("g")
-                                                .scope(InjectVariablesScope.LOCAL),
-                                        new VcsTagTask()
-                                                .description("${bamboo.g.gitTag}")
-                                                .tagName("${bamboo.g.gitTag}")
-                                                .defaultRepository())
-                                .dockerConfiguration(
-                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
-                                )
-                        )
-                )
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                .enabled(true), new InjectVariablesTask().description("Inject git Tag")
+                                .path("git.tag")
+                                .namespace("g")
+                                .scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)
 
                 .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement()
-                        .createForVcsBranch()
-                        .delete(new BranchCleanup()
-                                .whenInactiveInRepositoryAfterDays(14))
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
+                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
                         .notificationForCommitters());
     }
 
-    public Plan createSecBuild() {
-        return new Plan(
-                project(),
-                SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
-                .description("Security Analysis Plan")
-                .stages(new Stage("Default Stage")
-                        .jobs(new Job("Default Job",
-                                new BambooKey("JOB1"))
-                                .tasks(
-                                        new ScriptTask()
-                                                .description("Clean")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-                                                        "rm -rf ./*"),
-                                        new VcsCheckoutTask()
-                                                .description("Checkout Default Repository")
-                                                .cleanCheckout(true)
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
-                                        new ScriptTask()
-                                                .description("Sonar")
-                                                .location(Location.FILE)
-                                                .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
-                                                .argument(SERVICE_NAME))
-                                .dockerConfiguration(
-                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .dockerRunArguments("--net=host")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
-                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
-                                )
-                        )
-                )
+
+    public Plan createNightPlan() {
+
+        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
+                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
+                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                .enabled(true))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(
-                        new ScheduledTrigger()
-                                .scheduleOnceDaily(LocalTime.of(23, 00)))
-                .planBranchManagement(new PlanBranchManagement()
-                        .createForVcsBranchMatching("release.*")
-                        .notificationForCommitters());
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
     }
+
+
+    public Plan createSecBuild() {
+
+        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Sonar")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
+                                .argument(SERVICE_NAME))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .dockerRunArguments("--net=host")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
+    }
+
 }

bamboo-specs/src/test/java/buildjob/PlanSpecTest.java

@@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
 public class PlanSpecTest {
     @Test
     public void checkYourPlanOffline() throws PropertiesValidationException {
+
         Plan plan = new PlanSpec().createPlan();
         EntityPropertiesBuilders.build(plan);
 
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        EntityPropertiesBuilders.build(nightPlan);
+
         Plan secPlan = new PlanSpec().createSecBuild();
         EntityPropertiesBuilders.build(secPlan);
     }

redaction-service-image-v1/src/main/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM red/redaction-service-base-v1:1.0.0
+FROM red/redaction-service-base-v1:2.0.0
 
 ARG PLATFORM_JAR
 

redaction-service-v1/pom.xml

@@ -29,15 +29,10 @@
 
     <dependencyManagement>
         <dependencies>
-            <dependency>
-                <groupId>com.dslplatform</groupId>
-                <artifactId>dsl-json-java8</artifactId>
-                <version>${dsljson.version}</version>
-            </dependency>
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.13.0</version>
+                <version>1.20.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.166.0</persistence-service.version>
+        <persistence-service.version>1.254.0</persistence-service.version>
     </properties>
 
     <dependencies>

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedactions.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;
 
+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -11,6 +12,7 @@ import java.util.Map;
 
 @Data
 @Builder
+@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class ImportedRedactions {

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -38,6 +38,12 @@
             <version>${jackson.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.ahocorasick</groupId>
             <artifactId>ahocorasick</artifactId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/FloatFrequencyCounter.java

@@ -35,9 +35,7 @@ public class FloatFrequencyCounter {
     public Float getMostPopular() {
         Map.Entry<Float, Integer> mostPopular = null;
         for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null) {
-                mostPopular = entry;
-            } else if (entry.getValue() >= mostPopular.getValue()) {
+            if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
                 mostPopular = entry;
             }
         }
@@ -61,9 +59,7 @@ public class FloatFrequencyCounter {
     public Float getHighest() {
         Float highest = null;
         for (Float value : countPerValue.keySet()) {
-            if (highest == null) {
-                highest = value;
-            } else if (value > highest) {
+            if (highest == null || value > highest) {
                 highest = value;
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -27,11 +27,15 @@ public class SectionText {
     private boolean isTable;
     private String headline;
 
+    @Builder.Default
     private List<SectionArea> sectionAreas = new ArrayList<>();
+    @Builder.Default
     private Set<Image> images = new HashSet<>();
-
+    @Builder.Default
     private List<TextBlock> textBlocks = new ArrayList<>();
+    @Builder.Default
     private Map<String, CellValue> tabularData = new HashMap<>();
+    @Builder.Default
     private List<Integer> cellStarts = new ArrayList<>();
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import com.dslplatform.json.CompiledJson;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@CompiledJson
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedSectionText {
+
+    private int sectionNumber;
+    private String text;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java

@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.dslplatform.json.CompiledJson;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@CompiledJson
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedText {
+
+    private int numberOfPages;
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/StringFrequencyCounter.java

@@ -37,9 +37,7 @@ public class StringFrequencyCounter {
 
         Map.Entry<String, Integer> mostPopular = null;
         for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
-            if (mostPopular == null) {
-                mostPopular = entry;
-            } else if (entry.getValue() > mostPopular.getValue()) {
+            if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
                 mostPopular = entry;
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -23,16 +23,30 @@ public class TextBlock extends AbstractTextContainer {
 
     @Builder.Default
     private List<TextPositionSequence> sequences = new ArrayList<>();
+
+    @JsonIgnore
     private int rotation;
 
+    @JsonIgnore
     private String mostPopularWordFont;
+
+    @JsonIgnore
     private String mostPopularWordStyle;
+
+    @JsonIgnore
     private float mostPopularWordFontSize;
+
+    @JsonIgnore
     private float mostPopularWordHeight;
+
+    @JsonIgnore
     private float mostPopularWordSpaceWidth;
 
+    @JsonIgnore
     private float highestFontSize;
 
+
+    @JsonIgnore
     private String classification;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/MockMultipartFile.java

@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.client;
 
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
 import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 import org.springframework.util.FileCopyUtils;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-
 public class MockMultipartFile implements MultipartFile {
 
     private final String name;
@@ -82,13 +82,13 @@ public class MockMultipartFile implements MultipartFile {
     }
 
 
-    public byte[] getBytes() throws IOException {
+    public byte[] getBytes() {
 
         return this.content;
     }
 
 
-    public InputStream getInputStream() throws IOException {
+    public InputStream getInputStream() {
 
         return new ByteArrayInputStream(this.content);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java

@@ -5,8 +5,8 @@ import java.util.List;
 import java.util.Map;
 
 import com.dslplatform.json.CompiledJson;
+
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
@@ -16,6 +16,6 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class NerEntities {
 
-   private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
+    private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,29 +1,34 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.*;
-import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.*;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.resources.RedactionResource;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @RestController
@@ -37,6 +42,7 @@ public class RedactionController implements RedactionResource {
     private final RedactionLogMergeService redactionLogMergeService;
     private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
 
+
     @Override
     public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
 
@@ -118,14 +124,17 @@ public class RedactionController implements RedactionResource {
     @Override
     public void testRules(@RequestBody String rules) {
 
-        droolsExecutionService.testRules(rules);
+        try {
+            droolsExecutionService.testRules(rules);
+        } catch (Exception e) {
+            throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
+        }
     }
 
 
     @Override
     public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
 
-
         return redactionLogMergeService.provideRedactionLog(redactionRequest);
     }
 
@@ -134,19 +143,14 @@ public class RedactionController implements RedactionResource {
 
         try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
             document.save(byteArrayOutputStream);
-            return RedactionResult.builder()
-                    .document(byteArrayOutputStream.toByteArray())
-                    .numberOfPages(numberOfPages)
-                    .build();
+            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
         }
 
     }
 
 
     @Override
-    public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId,
-                                               @PathVariable("fileId") String fileId,
-                                               @RequestBody ManualRedactions manualRedactions) {
+    public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions) {
 
         var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
         log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/LegacyPDFStreamEngine.java

@@ -301,12 +301,24 @@ class LegacyPDFStreamEngine extends PDFStreamEngine
             nextY -= pageSize.getLowerLeftY();
         }
 
-        processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
-                pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
-                Math.abs(dyDisplay), dxDisplay,
-                Math.abs(spaceWidthDisplay), unicodeMapping, new int[] { code }, font,
-                fontSize,
-                (int)(fontSize * textMatrix.getScalingFactorX())));
+        // This is a hack for unicode letter with 2 chars e.g. RA see unicodeProblem.pdf
+        if(unicodeMapping.length() == 2){
+            processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
+                    pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
+                    Math.abs(dyDisplay), dxDisplay,
+                    Math.abs(spaceWidthDisplay), Character.toString(unicodeMapping.charAt(0)), new int[] { code }, font,
+                    fontSize,
+                    (int)(fontSize * textMatrix.getScalingFactorX())));
+            processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
+                    pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
+                    Math.abs(dyDisplay), dxDisplay,
+                    Math.abs(spaceWidthDisplay), Character.toString(unicodeMapping.charAt(1)), new int[] { code }, font,
+                    fontSize,
+                    (int)(fontSize * textMatrix.getScalingFactorX())));
+        } else {
+
+            processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(), pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY, Math.abs(dyDisplay), dxDisplay, Math.abs(spaceWidthDisplay), unicodeMapping, new int[]{code}, font, fontSize, (int) (fontSize * textMatrix.getScalingFactorX())));
+        }
     }
 
     /**

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java

@@ -187,7 +187,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 rulings.addAll(path);
             }
         } catch (UnsupportedOperationException e) {
-            log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor()
+            log.debug("UnsupportedOperationException: " + getGraphicsState().getStrokingColor()
                     .getColorSpace()
                     .getName() + " or " + getGraphicsState().getNonStrokingColor()
                     .getColorSpace()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java

@@ -1,30 +1,40 @@
 package com.iqser.red.service.redaction.v1.server.parsing.model;
 
+import org.apache.pdfbox.text.TextPosition;
+import org.springframework.beans.BeanUtils;
+
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
-import org.apache.pdfbox.text.TextPosition;
-import org.springframework.beans.BeanUtils;
 
 @Data
+@Builder
 @NoArgsConstructor
+@AllArgsConstructor
 @CompiledJson
 public class RedTextPosition {
 
     private String textMatrix;
+    private float[] position;
+
+    @JsonIgnore
     private int rotation;
-    private float y;
+
+    @JsonIgnore
     private float pageHeight;
+
+    @JsonIgnore
     private float pageWidth;
+
     private String unicode;
-    private float XDirAdj;
-    private float YDirAdj;
-    private float width;
-    private float heightDir;
-    private float widthDirAdj;
+
+    @JsonIgnore
     private float dir;
 
     // not used in reanalysis
@@ -53,8 +63,37 @@ public class RedTextPosition {
 
         pos.setTextMatrix(textPosition.getTextMatrix().toString());
 
+        var position = new float[4];
+
+        position[0] = textPosition.getXDirAdj();
+        position[1] = textPosition.getYDirAdj();
+        position[2] = textPosition.getWidthDirAdj();
+        position[3] = textPosition.getHeightDir();
+
+        pos.setPosition(position);
         return pos;
     }
 
 
+    @JsonIgnore
+    public float getXDirAdj(){
+        return position[0];
+    }
+
+    @JsonIgnore
+    public float getYDirAdj(){
+        return position[1];
+    }
+
+    @JsonIgnore
+    public float getWidthDirAdj(){
+        return position[2];
+    }
+
+    @JsonIgnore
+    public float getHeightDir(){
+        return position[3];
+    }
+
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextDirection.java

@@ -0,0 +1,62 @@
+package com.iqser.red.service.redaction.v1.server.parsing.model;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+import lombok.Getter;
+
+@Getter
+public enum TextDirection {
+    ZERO(0f),
+    QUARTER_CIRCLE(90f),
+    HALF_CIRCLE(180f),
+    THREE_QUARTER_CIRCLE(270f);
+
+    public static final String VALUE_STRING_SUFFIX = "°";
+
+    @JsonValue
+    private final float degrees;
+    private final float radians;
+
+    TextDirection(float degreeValue) {
+        degrees = degreeValue;
+        radians = (float) Math.toRadians(degreeValue);
+    }
+
+    @Override
+    public String toString() {
+        return degrees + VALUE_STRING_SUFFIX;
+    }
+
+    @com.dslplatform.json.JsonValue
+    public float jsonValue() {
+        return getDegrees();
+    }
+
+    @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+    public static TextDirection fromDegrees(float degrees) {
+
+        for (var dir : TextDirection.values()) {
+            if (degrees == dir.degrees) {
+                return dir;
+            }
+        }
+
+        throw new IllegalArgumentException(String.format("A value of %f is not supported by TextDirection", degrees));
+    }
+
+    public static TextDirection fromString(String degreesAsString) {
+
+        Objects.requireNonNull(degreesAsString, "Cannot construct a text direction from a null value");
+
+        String value = degreesAsString.strip();
+
+        if (degreesAsString.endsWith(VALUE_STRING_SUFFIX)) {
+            value = degreesAsString.replace(VALUE_STRING_SUFFIX + "$", "");
+        }
+
+        return fromDegrees(Float.parseFloat(value));
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -1,32 +1,45 @@
 package com.iqser.red.service.redaction.v1.server.parsing.model;
 
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.text.TextPosition;
+
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.pdfbox.text.TextPosition;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
 
 @Slf4j
 @Data
+@Builder
 @CompiledJson
 @NoArgsConstructor
+@AllArgsConstructor
 @JsonIgnoreProperties({"empty"})
 public class TextPositionSequence implements CharSequence {
 
+    public static final int HEIGHT_PADDING = 2;
     private int page;
     private List<RedTextPosition> textPositions = new ArrayList<>();
 
-    private float x1;
-    private float x2;
+    private TextDirection dir;
+    private int rotation;
+    private float pageHeight;
+    private float pageWidth;
+
 
     public TextPositionSequence(int page) {
 
@@ -34,7 +47,7 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public static TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
+    public TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
 
         var textPositionSequence = new TextPositionSequence();
         textPositionSequence.textPositions = textPositions;
@@ -48,6 +61,10 @@ public class TextPositionSequence implements CharSequence {
 
         this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
         this.page = page;
+        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
     }
 
 
@@ -78,7 +95,15 @@ public class TextPositionSequence implements CharSequence {
     @Override
     public TextPositionSequence subSequence(int start, int end) {
 
-        return fromData(textPositions.subList(start, end), page);
+        var textPositionSequence = new TextPositionSequence();
+        textPositionSequence.textPositions = textPositions.subList(start, end);
+        textPositionSequence.page = page;
+        textPositionSequence.dir = dir;
+        textPositionSequence.rotation = rotation;
+        textPositionSequence.pageHeight = pageHeight;
+        textPositionSequence.pageWidth = pageWidth;
+
+        return textPositionSequence;
     }
 
 
@@ -99,15 +124,26 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public void add(RedTextPosition textPosition) {
+    public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
 
         this.textPositions.add(textPosition);
+        this.page = textPositionSequence.getPage();
+        this.dir = textPositionSequence.getDir();
+        this.rotation = textPositionSequence.getRotation();
+        this.pageHeight = textPositionSequence.getPageHeight();
+        this.pageWidth = textPositionSequence.getPageWidth();
     }
 
 
     public void add(TextPosition textPosition) {
 
         this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
+
+        this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
+        this.rotation = textPositions.get(0).getRotation();
+        this.pageHeight = textPositions.get(0).getPageHeight();
+        this.pageWidth = textPositions.get(0).getPageWidth();
+
     }
 
 
@@ -115,7 +151,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getX1() {
 
-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
             return textPositions.get(0).getYDirAdj() - getTextHeight();
         } else {
             return textPositions.get(0).getXDirAdj();
@@ -127,11 +163,10 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getX2() {
 
-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
             return textPositions.get(0).getYDirAdj();
         } else {
-            return textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
+            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
         }
     }
 
@@ -140,15 +175,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getRotationAdjustedY() {
 
-        return textPositions.get(0).getY();
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getRotationAdjustedX() {
-
-        return textPositions.get(0).getXDirAdj();
+        return textPositions.get(0).getYDirAdj();
     }
 
 
@@ -156,10 +183,10 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getY1() {
 
-        if (textPositions.get(0).getRotation() == 90) {
+        if (rotation == 90) {
             return textPositions.get(0).getXDirAdj();
         } else {
-            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj();
+            return pageHeight - textPositions.get(0).getYDirAdj();
         }
     }
 
@@ -168,10 +195,10 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getY2() {
 
-        if (textPositions.get(0).getRotation() == 90) {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - 2;
+        if (rotation == 90) {
+            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - HEIGHT_PADDING;
         } else {
-            return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() + getTextHeight();
+            return pageHeight - textPositions.get(0).getYDirAdj() + getTextHeight();
         }
     }
 
@@ -180,7 +207,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getTextHeight() {
 
-        return textPositions.get(0).getHeightDir() + 2;
+        return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
     }
 
 
@@ -245,86 +272,39 @@ public class TextPositionSequence implements CharSequence {
 
     @JsonIgnore
     @JsonAttribute(ignore = true)
-    public int getRotation() {
-
-        return textPositions.get(0).getRotation();
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
+    @SneakyThrows
     public Rectangle getRectangle() {
 
-        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0).getRotation(), textPositions.get(0).getDir());
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, rotation, dir);
 
-        float height = getTextHeight();
+        float textHeight = getTextHeight();
 
-        float posXInit = getX1();
-        float posXEnd;
-        float posYInit;
-        float posYEnd;
+        RedTextPosition firstTextPos = textPositions.get(0);
+        RedTextPosition lastTextPos = textPositions.get(textPositions.size() - 1);
 
-        if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() != 0.0f) {
-            posXEnd = textPositions.get(0).getYDirAdj() + 2;
-            posYInit = getY1();
-            posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
-
-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 270.0f) {
-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
-                    .getWidth() - textPositions.get(textPositions.size() - 1).getWidth() - 1;
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
-
-        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) {
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = posYInit + 1;
-            posXInit = textPositions.get(0).getXDirAdj();
-            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
-
-        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) {
-            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
-
-        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 0.0f) {
-            posXInit = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
-            posXEnd = textPositions.get(0).getXDirAdj();
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
-
-        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 90f) {
-            posYInit = getX1();
-            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
-                    .getWidthDirAdj() - 3;
-            posXInit = textPositions.get(0).getYDirAdj() + 2;
-            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
-
-        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
-            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
-                    .getWidthDirAdj() - 3;
-            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
-            posYEnd = textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() - height + 2;
+        Point2D bottomLeft = new Point2D.Double(firstTextPos.getXDirAdj(), firstTextPos.getYDirAdj() - HEIGHT_PADDING);
+        Point2D topRight = new Point2D.Double(lastTextPos.getXDirAdj() + lastTextPos.getWidthDirAdj(), lastTextPos.getYDirAdj() + textHeight + HEIGHT_PADDING);
 
+        AffineTransform transform = new AffineTransform();
+        if (dir == TextDirection.ZERO || dir == TextDirection.HALF_CIRCLE) {
+            transform.rotate(dir.getRadians(), pageWidth / 2f, pageHeight / 2f);
+            transform.translate(0f, pageHeight + textHeight);
+            transform.scale(1., -1.);
+        } else if (dir == TextDirection.QUARTER_CIRCLE) {
+            transform.rotate(dir.getRadians(), pageWidth / 2f, pageWidth / 2f);
+            transform.translate(0f, pageWidth + textHeight);
+            transform.scale(1., -1.);
         } else {
-            posXEnd = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
-            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
+            transform.rotate(dir.getRadians(), pageHeight / 2f, pageHeight / 2f);
+            transform.translate(0f, pageWidth + textHeight);
+            transform.scale(1., -1.);
         }
 
-        var rectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
-        log.debug("Rectangle: {}", rectangle);
-        return rectangle;
+        bottomLeft = transform.transform(bottomLeft, null);
+        topRight = transform.transform(topRight, null);
+
+        return new Rectangle( //
+                new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), page);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,5 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@@ -8,13 +14,9 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.stereotype.Service;
-
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
 
 @Slf4j
 @Service
@@ -55,6 +57,9 @@ public class RedactionMessageReceiver {
                     result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
                     log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
                     break;
+
+                default:
+                    throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
             }
 
             result.setMessageType(analyzeRequest.getMessageType());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -2,9 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.Data;
-import lombok.EqualsAndHashCode;
+import lombok.*;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -12,12 +10,16 @@ import java.util.List;
 import java.util.Set;
 
 @Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity implements ReasonHolder {
 
     private String word;
     private String type;
     private boolean redaction;
+    private boolean falsePositive;
     private String redactionReason;
     private String legalBasis;
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
@@ -50,6 +52,8 @@ public class Entity implements ReasonHolder {
 
     private EntityType entityType;
 
+    private boolean resized;
+
 
     public Entity(String word, String type, boolean redaction, String redactionReason,
                   List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java

@@ -1,15 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
-
 import java.util.ArrayList;
 import java.util.List;
 
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+
 @Data
+@Builder
 @RequiredArgsConstructor
 @AllArgsConstructor
 @EqualsAndHashCode

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java

@@ -13,6 +13,7 @@ public class PdfImage {
     @NonNull
     private ImageType imageType;
     private boolean isAppendedToParagraph;
+    @NonNull
     private boolean hasTransparency;
     @NonNull
     private int page;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java

@@ -1,14 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
+@Builder
 @CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -9,7 +9,11 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Getter;
 
 public class SearchableText {
 
@@ -17,6 +21,7 @@ public class SearchableText {
     @JsonIgnore
     @JsonAttribute(ignore = true)
     private transient String stringRepresentation;
+    @Getter
     private final List<TextPositionSequence> sequences = new ArrayList<>();
 
 
@@ -100,7 +105,7 @@ public class SearchableText {
                                     .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                             .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
                             .charAt(j, caseInsensitive) != ' ') {
-                        partMatch.add(searchSpace.get(i).textPositionAt(j));
+                        partMatch.add(searchSpace.get(i),searchSpace.get(i).textPositionAt(j));
                         if (!(j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
                                 .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                             counter++;
@@ -194,7 +199,7 @@ public class SearchableText {
     }
 
 
-    public String buildString(List<TextPositionSequence> sequences) {
+    public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();
 
@@ -218,6 +223,31 @@ public class SearchableText {
     }
 
 
+    public String getAsStringWithLinebreaksSorted() {
+
+
+        var sorted = sequences.stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
+
+        StringBuilder sb = new StringBuilder();
+
+        TextPositionSequence previous = null;
+        for (TextPositionSequence word : sorted) {
+
+            if (previous != null) {
+                if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
+                    sb.append('\n');
+                } else {
+                    sb.append(' ');
+                }
+            }
+            sb.append(word.toString());
+            previous = word;
+        }
+        return sb.append("\n").toString();
+    }
+
+
+
     public String getAsStringWithLinebreaks() {
 
         StringBuilder sb = new StringBuilder();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1,28 +1,38 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-
-import com.iqser.red.service.redaction.v1.model.ArgumentType;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
-import lombok.Builder;
-import lombok.Data;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.ArgumentType;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
 @Data
 @Slf4j
 @Builder
@@ -61,15 +71,23 @@ public class Section {
     @Builder.Default
     private List<FileAttribute> fileAttributes = new ArrayList<>();
 
+    @Builder.Default
+    private List<SectionArea> sectionAreas = new ArrayList<>();
+
+    private ManualRedactions manualRedactions;
+
 
     @SuppressWarnings("unused")
-    @WhenCondition
+    @ThenAction
     public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
 
-        Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
+        Set<Entity> entitiesOfType = nerEntities.stream()
+                .filter(nerEntity -> nerEntity.getType().equals(type))
+                .collect(Collectors.toSet());
         List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
         Set<Entity> found = EntitySearchUtils.findEntities(searchText, new SearchImplementation(values, dictionary.isCaseInsensitiveDictionary(asType)), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
-        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
+        EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        found = found.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
 
         Set<Entity> finalResult = new HashSet<>();
 
@@ -93,15 +111,21 @@ public class Section {
         nerEntities.removeAll(entitiesOfType);
     }
 
+
     @SuppressWarnings("unused")
-    @WhenCondition
-    public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType, @Argument(ArgumentType.TYPE) String combineTypes,
-                               @Argument(ArgumentType.INTEGER) int maxDistanceBetween, @Argument(ArgumentType.TYPE) String asType,
-                               @Argument(ArgumentType.INTEGER) int minPartMatches, @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) {
+    @ThenAction
+    public void combineAiTypes(@Argument(ArgumentType.TYPE) String startType,
+                               @Argument(ArgumentType.TYPE) String combineTypes,
+                               @Argument(ArgumentType.INTEGER) int maxDistanceBetween,
+                               @Argument(ArgumentType.TYPE) String asType,
+                               @Argument(ArgumentType.INTEGER) int minPartMatches,
+                               @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) {
 
         Set<String> combineSet = Set.of(combineTypes.split(","));
 
-        List<Entity> sorted = nerEntities.stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
+        List<Entity> sorted = nerEntities.stream()
+                .sorted(Comparator.comparing(Entity::getStart))
+                .collect(Collectors.toList());
         Set<Entity> found = new HashSet<>();
         int start = -1;
         int lastEnd = -1;
@@ -158,49 +182,67 @@ public class Section {
         }
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
+    public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
+                                           @Argument(ArgumentType.STRING) String value) {
 
-        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()));
+        return fileAttributes != null && fileAttributes.stream()
+                .anyMatch(attribute -> id.equals(attribute.getId()) && value.equals(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {
+    public boolean fileAttributeByPlaceholderEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
+                                                    @Argument(ArgumentType.STRING) String value) {
 
-        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()));
+        return fileAttributes != null && fileAttributes.stream()
+                .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equals(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {
+    public boolean fileAttributeByLabelEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
+                                              @Argument(ArgumentType.STRING) String value) {
 
-        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
+        return fileAttributes != null && fileAttributes.stream()
+                .anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
+    public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
+                                                     @Argument(ArgumentType.STRING) String value) {
 
-        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()));
+        return fileAttributes != null && fileAttributes.stream()
+                .anyMatch(attribute -> id.equals(attribute.getId()) && value.equalsIgnoreCase(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder, @Argument(ArgumentType.STRING) String value) {
+    public boolean fileAttributeByPlaceholderEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String placeholder,
+                                                              @Argument(ArgumentType.STRING) String value) {
 
         return fileAttributes != null && fileAttributes.stream()
                 .anyMatch(attribute -> placeholder.equals(attribute.getPlaceholder()) && value.equalsIgnoreCase(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) String value) {
+    public boolean fileAttributeByLabelEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String label,
+                                                        @Argument(ArgumentType.STRING) String value) {
 
-        return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()));
+        return fileAttributes != null && fileAttributes.stream()
+                .anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equalsIgnoreCase(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean hasTableHeader(@Argument(ArgumentType.STRING) String headerName) {
@@ -209,6 +251,7 @@ public class Section {
         return tabularData != null && tabularData.containsKey(cleanHeaderName);
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean aiMatchesType(@Argument(ArgumentType.TYPE) String type) {
@@ -216,6 +259,7 @@ public class Section {
         return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
@@ -223,6 +267,7 @@ public class Section {
         return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
@@ -230,6 +275,7 @@ public class Section {
         return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean headlineContainsWord(@Argument(ArgumentType.STRING) String word) {
@@ -237,9 +283,11 @@ public class Section {
         return StringUtils.containsIgnoreCase(headline, word);
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) {
+    public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx,
+                                 @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive) {
 
         var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive);
 
@@ -248,19 +296,26 @@ public class Section {
         return matcher.find();
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
-    public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) {
+    public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName,
+                             @Argument(ArgumentType.STRING) String value) {
 
         String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
 
-        return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value);
+        return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
+                .toString()
+                .equals(value);
     }
 
-    @SuppressWarnings("unused")
+
     @ThenAction
-    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
-                                    @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
+    @SuppressWarnings("unused")
+    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type,
+                                    @Argument(ArgumentType.REGEX) String prefixPattern,
+                                    @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                    @Argument(ArgumentType.INTEGER) int group) {
 
         expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null);
     }
@@ -268,11 +323,15 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
-                                    @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
+    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type,
+                                    @Argument(ArgumentType.REGEX) String prefixPattern,
+                                    @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                    @Argument(ArgumentType.INTEGER) int group,
                                     @Argument(ArgumentType.REGEX) String valuePattern) {
 
-        if (StringUtils.isEmpty(prefixPattern)) return;
+        if (StringUtils.isEmpty(prefixPattern)) {
+            return;
+        }
 
         var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
         var compiledPrefixPattern = Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive);
@@ -317,8 +376,10 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
+    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
+                              @Argument(ArgumentType.REGEX) String suffixPattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                              @Argument(ArgumentType.INTEGER) int group) {
 
         expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null);
     }
@@ -326,11 +387,15 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
+    public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
+                              @Argument(ArgumentType.REGEX) String suffixPattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                              @Argument(ArgumentType.INTEGER) int group,
                               @Argument(ArgumentType.REGEX) String valuePattern) {
 
-        if (StringUtils.isEmpty(suffixPattern)) return;
+        if (StringUtils.isEmpty(suffixPattern)) {
+            return;
+        }
 
         var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
         var compiledSuffixPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive);
@@ -374,7 +439,9 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+    public void redactImage(@Argument(ArgumentType.TYPE) String type,
+                            @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                            @Argument(ArgumentType.STRING) String reason,
                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         redactImage(type, ruleNumber, reason, legalBasis, true);
@@ -383,7 +450,9 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotImage(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
+    public void redactNotImage(@Argument(ArgumentType.TYPE) String type,
+                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                               @Argument(ArgumentType.STRING) String reason) {
 
         redactImage(type, ruleNumber, reason, null, false);
     }
@@ -391,7 +460,8 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+    public void redact(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                       @Argument(ArgumentType.STRING) String reason,
                        @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         redact(type, ruleNumber, reason, legalBasis, true);
@@ -400,7 +470,8 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
+    public void redactNot(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                          @Argument(ArgumentType.STRING) String reason) {
 
         redact(type, ruleNumber, reason, null, false);
     }
@@ -408,8 +479,10 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason,
+    public void redactLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType,
+                                @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                @Argument(ArgumentType.STRING) String reason,
                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
@@ -418,8 +491,11 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                   @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) {
+    public void redactNotLineAfter(@Argument(ArgumentType.STRING) String start,
+                                   @Argument(ArgumentType.TYPE) String asType,
+                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                   @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                   @Argument(ArgumentType.STRING) String reason) {
 
         redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false);
 
@@ -428,9 +504,12 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                              @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                              @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void redactByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                              @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
     }
@@ -438,21 +517,46 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                 @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+    public void redactNotByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                                 @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                 @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                  @Argument(ArgumentType.STRING) String reason) {
 
         redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
     }
 
 
+    @Deprecated
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                              @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true);
+    }
+
+
     @ThenAction
     @SuppressWarnings("unused")
     public void redactBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                              @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                              @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true);
+    }
+
+
+    @Deprecated
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
+                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
+                                 @Argument(ArgumentType.STRING) String reason) {
+
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, false, reason, null, false);
     }
 
 
@@ -460,9 +564,9 @@ public class Section {
     @SuppressWarnings("unused")
     public void redactNotBetween(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.STRING) String stop, @Argument(ArgumentType.TYPE) String asType,
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                                 @Argument(ArgumentType.STRING) String reason) {
+                                 @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason) {
 
-        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
+        redactBetween(start, stop, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false);
     }
 
 
@@ -488,8 +592,10 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
-                           @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason,
+    public void redactCell(@Argument(ArgumentType.STRING) String cellHeader,
+                           @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
+                           @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
+                           @Argument(ArgumentType.STRING) String reason,
                            @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
@@ -498,8 +604,11 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type,
-                              @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations, @Argument(ArgumentType.STRING) String reason) {
+    public void redactNotCell(@Argument(ArgumentType.STRING) String cellHeader,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.TYPE) String type,
+                              @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
+                              @Argument(ArgumentType.STRING) String reason) {
 
         annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null);
     }
@@ -507,9 +616,13 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                          @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                          @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void redactAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                                          @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                          @Argument(ArgumentType.INTEGER) int group,
+                                          @Argument(ArgumentType.TYPE) String asType,
+                                          @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                          @Argument(ArgumentType.STRING) String reason,
+                                          @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
     }
@@ -517,9 +630,12 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                             @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType,
-                                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
+    public void redactNotAndRecommendByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                                             @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                             @Argument(ArgumentType.INTEGER) int group,
+                                             @Argument(ArgumentType.TYPE) String asType,
+                                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                             @Argument(ArgumentType.STRING) String reason) {
 
         redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
     }
@@ -527,8 +643,10 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                         @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {
+    public void addRecommendationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                                         @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                         @Argument(ArgumentType.INTEGER) int group,
+                                         @Argument(ArgumentType.TYPE) String asType) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -537,7 +655,7 @@ public class Section {
         while (matcher.find()) {
             String match = matcher.group(group);
             if (StringUtils.isNotBlank(match) && match.length() >= 3) {
-                localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
+                localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(match);
             }
         }
     }
@@ -545,11 +663,14 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
-                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason) {
+    public void redactNotAndReference(@Argument(ArgumentType.TYPE) String type,
+                                      @Argument(ArgumentType.REFERENCE_TYPE) String referenceType,
+                                      @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                      @Argument(ArgumentType.STRING) String reason) {
 
-
-        Set<Entity> references = entities.stream().filter(entity -> entity.getType().equals(referenceType)).collect(Collectors.toSet());
+        Set<Entity> references = entities.stream()
+                .filter(entity -> entity.getType().equals(referenceType))
+                .collect(Collectors.toSet());
 
         entities.forEach(entity -> {
             if (entity.getType().equals(type)) {
@@ -564,8 +685,11 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix, @Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                   @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void redactIfPrecededBy(@Argument(ArgumentType.STRING) String prefix,
+                                   @Argument(ArgumentType.TYPE) String type,
+                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                   @Argument(ArgumentType.STRING) String reason,
+                                   @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         entities.forEach(entity -> {
             if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) {
@@ -580,13 +704,16 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                             @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+    public void addRedaction(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType,
+                             @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                             @Argument(ArgumentType.STRING) String reason,
+                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
 
         Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
         EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
     }
 
+
     @ThenAction
     @SuppressWarnings("unused")
     public void ignore(@Argument(ArgumentType.TYPE) String type) {
@@ -594,6 +721,7 @@ public class Section {
         entities.removeIf(entity -> entity.getType().equals(type) && entity.getEntityType().equals(EntityType.ENTITY));
     }
 
+
     @ThenAction
     @SuppressWarnings("unused")
     public void ignoreRecommendations(@Argument(ArgumentType.TYPE) String type) {
@@ -604,8 +732,10 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.STRING) String pattern,
-                                             @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
+    public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type,
+                                             @Argument(ArgumentType.STRING) String pattern,
+                                             @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                             @Argument(ArgumentType.INTEGER) int group) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -634,8 +764,10 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                         @Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.TYPE) String asType) {
+    public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
+                                         @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                         @Argument(ArgumentType.INTEGER) int group,
+                                         @Argument(ArgumentType.TYPE) String asType) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -653,7 +785,8 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) {
+    public void addHintAnnotation(@Argument(ArgumentType.STRING) String value,
+                                  @Argument(ArgumentType.TYPE) String asType) {
 
         Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false);
         EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
@@ -662,7 +795,8 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void recommendLineAfter(@Argument(ArgumentType.STRING) String start, @Argument(ArgumentType.TYPE) String asType) {
+    public void recommendLineAfter(@Argument(ArgumentType.STRING) String start,
+                                   @Argument(ArgumentType.TYPE) String asType) {
 
         String[] values = StringUtils.substringsBetween(text, start, "\n");
 
@@ -678,7 +812,7 @@ public class Section {
                 }
 
                 if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) {
-                    localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(cleanValue);
+                    localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(cleanValue);
                 }
             }
         }
@@ -687,12 +821,67 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) {
+    public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.TYPE) String type) {
 
         annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
     }
 
 
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSectionText(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+                                  @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                                 @Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true);
+    }
+
+
+    @ThenAction
+    @SuppressWarnings("unused")
+    public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason,
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+
+        for (SectionArea sectionArea : sectionAreas) {
+
+            RedRectangle2D position = RedRectangle2D.builder()
+                    .height(sectionArea.getHeight() + 4)
+                    .width(sectionArea.getWidth() + 4)
+                    .x(sectionArea.getTopLeft().getX() - 2)
+                    .y(sectionArea.getTopLeft().getY() - 2)
+                    .build();
+
+            log.debug("SectionArea: {}", sectionArea);
+            log.debug("Position {}", position.toString());
+
+            Image image = Image.builder()
+                    .page(sectionArea.getPage())
+                    .position(position)
+                    .redaction(true)
+                    .hasTransparency(false)
+                    .sectionNumber(sectionNumber)
+                    .section(headline)
+                    .matchedRule(ruleNumber)
+                    .legalBasis(legalBasis)
+                    .redactionReason(reason)
+                    .type(type)
+                    .build();
+
+            images.add(image);
+        }
+    }
+
+
     private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis,
                                            boolean redaction) {
 
@@ -703,17 +892,18 @@ public class Section {
             if (StringUtils.isNotBlank(match) && match.length() >= 3) {
                 Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
-                localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
+                localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(match);
             }
         }
     }
 
 
-    private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted, int ruleNumber, String reason, String legalBasis, Engine engine, boolean asRecommendation) {
+    private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
+                                     int ruleNumber, String reason, String legalBasis, Engine engine,
+                                     boolean asRecommendation) {
 
         String text = caseInsensitive ? searchText.toLowerCase() : searchText;
-        Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType),
-                new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+        Set<Entity> found = EntitySearchUtils.findEntities(text, new SearchImplementation(value, caseInsensitive), dictionary.getType(asType), new FindEntityDetails(asType, headline, sectionNumber, false, false, engine, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY));
         found.forEach(entity -> {
             if (redacted) {
                 entity.setRedaction(true);
@@ -723,7 +913,8 @@ public class Section {
             }
         });
 
-        return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return cleared.stream().filter(f -> !f.isFalsePositive()).collect(Collectors.toSet());
     }
 
 
@@ -753,7 +944,8 @@ public class Section {
     }
 
 
-    private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) {
+    private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact,
+                              boolean addAsRecommendations, String reason, String legalBasis) {
 
         String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
 
@@ -776,7 +968,8 @@ public class Section {
 
             Set<Entity> singleEntitySet = new HashSet<>();
             singleEntitySet.add(entity);
-            EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary);
+
+            EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary, manualRedactions);
 
             EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
 
@@ -800,7 +993,8 @@ public class Section {
     }
 
 
-    private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
+    private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
+                                 String legalBasis, boolean redaction) {
 
         String[] values = StringUtils.substringsBetween(text, start, "\n");
 
@@ -811,7 +1005,7 @@ public class Section {
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                     if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
                     }
                 }
             }
@@ -819,7 +1013,31 @@ public class Section {
     }
 
 
-    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) {
+
+    public void redactLineAfterAcrossColumns(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
+                                 String legalBasis) {
+
+
+        String[] values = StringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
+
+        if (values != null) {
+            for (String value : values) {
+                if (StringUtils.isNotBlank(value)) {
+                    Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
+
+                    if (redactEverywhere && !isLocal()) {
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
+                    }
+                }
+            }
+        }
+    }
+
+
+
+    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
+                               String reason, String legalBasis, boolean redaction) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -835,9 +1053,24 @@ public class Section {
     }
 
 
-    private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
+    private void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, boolean excludeHeadLine, String reason, String legalBasis,
+                               boolean redaction) {
 
-        String[] values = StringUtils.substringsBetween(searchText, start, stop);
+        String[] values = new String[1];
+
+        if (start.isEmpty() && stop.isEmpty()) {
+            if (excludeHeadLine) {
+                values[0] = StringUtils.substringAfter(searchText, headline);
+            } else {
+                values[0] = searchText;
+            }
+        } else if (start.isEmpty() && searchText.contains(stop)) {
+            values[0] = StringUtils.substringBefore(searchText, stop);
+        } else if (stop.isEmpty() && searchText.contains(start)) {
+            values[0] = StringUtils.substringAfter(searchText, start);
+        } else {
+            values = StringUtils.substringsBetween(searchText, start, stop);
+        }
 
         if (values != null) {
             for (String value : values) {
@@ -847,7 +1080,7 @@ public class Section {
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                     if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
                     }
                 }
             }
@@ -855,7 +1088,8 @@ public class Section {
     }
 
 
-    private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
+    private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
+                                    String reason, String legalBasis, boolean redaction) {
 
         String[] values = StringUtils.substringsBetween(text, start, stop);
 
@@ -874,7 +1108,7 @@ public class Section {
                         EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                         if (redactEverywhere && !isLocal()) {
-                            localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
+                            localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(line.trim());
                         }
                     }
                 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import java.util.List;
+
 import lombok.AllArgsConstructor;
 import lombok.Data;
 
@@ -9,5 +11,6 @@ public class SectionSearchableTextPair {
 
     private Section section;
     private SearchableText searchableText;
+    private List<Integer> cellStarts;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageMetadata.java

@@ -11,4 +11,5 @@ public class ImageMetadata {
     private Position position;
     private Geometry geometry;
     private Filters filters;
+    private boolean alpha;
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java

@@ -21,4 +21,11 @@ public class ImageServiceResponse {
     @JsonAttribute(alternativeNames = {"imageMetadata"})
     private List<ImageMetadata> data = new ArrayList<>();
 
+    private List<ImageMetadata> dataCV = new ArrayList<>();
+
+    @JsonProperty(value = "imageMetadata")
+    @JsonAlias("data")
+    @JsonAttribute(alternativeNames = {"imageMetadata"})
+    public void setData(List<ImageMetadata> data) {this.data = data;}
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -1,21 +1,54 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+import org.springframework.web.bind.annotation.RequestBody;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedSectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
@@ -27,15 +60,6 @@ import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-import org.springframework.web.bind.annotation.RequestBody;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
 
 @Slf4j
 @Service
@@ -57,7 +81,7 @@ public class AnalyzeService {
     private final ImportedRedactionService importedRedactionService;
 
 
-    @Timed("analyzeDocumentStructure")
+    @Timed("redactmanager_analyzeDocumentStructure")
     public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
 
         long startTime = System.currentTimeMillis();
@@ -92,8 +116,9 @@ public class AnalyzeService {
                         .map(SectionArea::getPage)
                         .collect(Collectors.toSet()), sectionText.getSectionAreas())));
 
-        log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        log.info("Store text, simplified text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
 
         return AnalyzeResult.builder()
@@ -106,7 +131,7 @@ public class AnalyzeService {
     }
 
 
-    @Timed("reanalyze")
+    @Timed("redactmanager_reanalyze")
     @SneakyThrows
     public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
 
@@ -120,14 +145,10 @@ public class AnalyzeService {
             return analyze(analyzeRequest);
         }
 
-//        var dis = System.currentTimeMillis();
         DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
-//        log.info("Dictionary Increment time time: {} ms", (System.currentTimeMillis() - dis));
 
-//        var fis = System.currentTimeMillis();
         Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse()
                 .isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest);
-//        log.info("Find sections time: {} ms", (System.currentTimeMillis() - fis));
 
         if (sectionsToReanalyse.isEmpty()) {
             return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
@@ -145,36 +166,26 @@ public class AnalyzeService {
                 .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
                 .collect(Collectors.toList());
 
-//        long kis = System.currentTimeMillis();
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
-//        log.info("Kie time: {} ms", (System.currentTimeMillis() - kis));
 
-//        long dds = System.currentTimeMillis();
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-//        log.info("Dict Time time: {} ms", (System.currentTimeMillis() - dds));
 
-//        long pis = System.currentTimeMillis();
         PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
-//        log.info("Find Entities time: {}", (System.currentTimeMillis() - pis));
 
-//        long crs = System.currentTimeMillis();
         var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
-//        log.info("Create Redaction-log time: {} ms", (System.currentTimeMillis() - crs));
-//        long prs = System.currentTimeMillis();
+
         var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newRedactionLogEntries, false);
-//        log.info("Process imports time: {} ms", (System.currentTimeMillis() - prs));
+
         redactionLog.getRedactionLogEntry()
                 .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType()
                         .equals(IMPORTED_REDACTION_TYPE));
         redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
 
-//        var fls = System.currentTimeMillis();
-        //        log.info("Finalize time: {} ms", (System.currentTimeMillis() - fls));
         return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
     }
 
 
-    @Timed("analyze")
+    @Timed("redactmanager_analyze")
     public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
 
         long startTime = System.currentTimeMillis();
@@ -207,6 +218,7 @@ public class AnalyzeService {
     }
 
 
+    @Timed("redactmanager_findSectionsToReanalyse")
     private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog,
                                                  Text text, AnalyzeRequest analyzeRequest) {
 
@@ -224,7 +236,6 @@ public class AnalyzeService {
             }
         }
 
-//        long ss = System.currentTimeMillis();
 
         var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream()
                 .map(DictionaryIncrementValue::getValue).collect(Collectors.toList()), true);
@@ -236,7 +247,6 @@ public class AnalyzeService {
             }
 
         }
-//        log.info("Section Find time: {}", (System.currentTimeMillis() - ss));
 
         log.info("Should reanalyze {} sections for request: {}, took: {}", sectionsToReanalyse.size(), analyzeRequest, System.currentTimeMillis() - start);
 
@@ -282,7 +292,9 @@ public class AnalyzeService {
             return new HashSet<>();
         }
 
-        return Stream.concat(manualRedactions.getLegalBasisChanges()
+        return Stream.concat(manualRedactions.getResizeRedactions()
+                .stream()
+                .map(ManualResizeRedaction::getAnnotationId), Stream.concat(manualRedactions.getLegalBasisChanges()
                 .stream()
                 .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
                 .stream()
@@ -290,7 +302,7 @@ public class AnalyzeService {
                 .stream()
                 .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions()
                 .stream()
-                .map(ManualForceRedaction::getAnnotationId)))).collect(Collectors.toSet());
+                .map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
     }
 
     public List<RedactionLogLegalBasis> convert(List<LegalBasis> legalBasis) {
@@ -324,4 +336,17 @@ public class AnalyzeService {
         }
     }
 
+
+    private SimplifiedText convert(Text text) {
+
+        List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+        int numberOfPages = 0;
+
+        if (text != null) {
+            numberOfPages = text.getNumberOfPages();
+            text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText())));
+        }
+        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -6,6 +6,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import feign.FeignException;
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -17,18 +18,19 @@ import java.util.List;
 import java.util.*;
 import java.util.stream.Collectors;
 
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class DictionaryService {
 
+    private static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
     private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
     private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
 
 
+    @Timed("redactmanager_updateDictionary")
     public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
 
         log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
@@ -44,11 +46,16 @@ public class DictionaryService {
             updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
         }
 
-        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
+        return DictionaryVersion.builder()
+                .dossierTemplateVersion(dossierTemplateDictionaryVersion)
+                .dossierVersion(dossierDictionaryVersion)
+                .build();
     }
 
 
-    public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
+    @Timed("redactmanager_getDictionaryIncrements")
+    public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion,
+                                                       String dossierId) {
 
         DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
 
@@ -102,67 +109,78 @@ public class DictionaryService {
         try {
             DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
 
-            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, false)
-                    : dictionaryClient.getAllTypesForDossier(dossierId, false);
+            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, false) : dictionaryClient.getAllTypesForDossier(dossierId, false);
             if (CollectionUtils.isNotEmpty(typeResponse)) {
 
-                List<DictionaryModel> dictionary = typeResponse
-                        .stream()
-                        .map(t -> {
+                List<DictionaryModel> dictionary = typeResponse.stream().map(t -> {
 
-                            Optional<DictionaryModel> oldModel;
-                            if (dossierId == null) {
-                                var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
-                                oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
-                            } else {
-                                var representation = dictionariesByDossier.get(dossierId);
-                                oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
-                            }
+                    Optional<DictionaryModel> oldModel;
+                    if (dossierId == null) {
+                        var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
+                        oldModel = representation != null ? representation.getDictionary()
+                                .stream()
+                                .filter(f -> f.getType().equals(t.getType()))
+                                .findAny() : Optional.empty();
+                    } else {
+                        var representation = dictionariesByDossier.get(dossierId);
+                        oldModel = representation != null ? representation.getDictionary()
+                                .stream()
+                                .filter(f -> f.getType().equals(t.getType()))
+                                .findAny() : Optional.empty();
+                    }
 
-                            Set<DictionaryEntry> entries = new HashSet<>();
-                            Set<DictionaryEntry> falsePositives = new HashSet<>();
-                            Set<DictionaryEntry> falseRecommendations = new HashSet<>();
+                    Set<DictionaryEntry> entries = new HashSet<>();
+                    Set<DictionaryEntry> falsePositives = new HashSet<>();
+                    Set<DictionaryEntry> falseRecommendations = new HashSet<>();
 
-                            DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
+                    DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
 
-                            var newValues = newEntries.getEntries().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
-                            var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
-                            var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
+                    var newValues = newEntries.getEntries()
+                            .stream()
+                            .map(DictionaryEntry::getValue)
+                            .collect(Collectors.toSet());
+                    var newFalsePositivesValues = newEntries.getFalsePositives()
+                            .stream()
+                            .map(DictionaryEntry::getValue)
+                            .collect(Collectors.toSet());
+                    var newFalseRecommendationsValues = newEntries.getFalseRecommendations()
+                            .stream()
+                            .map(DictionaryEntry::getValue)
+                            .collect(Collectors.toSet());
 
+                    oldModel.ifPresent(oldDictionaryModel -> {
 
-                            oldModel.ifPresent(oldDictionaryModel -> {
+                    });
+                    // add old entries from existing DictionaryModel
+                    oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries()
+                            .stream()
+                            .filter(f -> !newValues.contains(f.getValue()))
+                            .collect(Collectors.toList())));
+                    oldModel.ifPresent(dictionaryModel -> falsePositives.addAll(dictionaryModel.getFalsePositives()
+                            .stream()
+                            .filter(f -> !newFalsePositivesValues.contains(f.getValue()))
+                            .collect(Collectors.toList())));
+                    oldModel.ifPresent(dictionaryModel -> falseRecommendations.addAll(dictionaryModel.getFalseRecommendations()
+                            .stream()
+                            .filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
+                            .collect(Collectors.toList())));
 
-                            });
-                            // add old entries from existing DictionaryModel
-                            oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(
-                                    f -> !newValues.contains(f.getValue())).collect(Collectors.toList())
-                            ));
-                            oldModel.ifPresent(dictionaryModel -> falsePositives.addAll(dictionaryModel.getFalsePositives().stream().filter(
-                                    f -> !newFalsePositivesValues.contains(f.getValue())).collect(Collectors.toList())
-                            ));
-                            oldModel.ifPresent(dictionaryModel -> falseRecommendations.addAll(dictionaryModel.getFalseRecommendations().stream().filter(
-                                    f -> !newFalseRecommendationsValues.contains(f.getValue())).collect(Collectors.toList())
-                            ));
+                    // Add Increments
+                    entries.addAll(newEntries.getEntries());
+                    falsePositives.addAll(newEntries.getFalsePositives());
+                    falseRecommendations.addAll(newEntries.getFalseRecommendations());
 
-                            // Add Increments
-                            entries.addAll(newEntries.getEntries());
-                            falsePositives.addAll(newEntries.getFalsePositives());
-                            falseRecommendations.addAll(newEntries.getFalseRecommendations());
-
-                            return new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
-                                    .isHint(), entries, falsePositives, falseRecommendations, dossierId != null);
-                        })
-                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
-                        .collect(Collectors.toList());
+                    return new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t.isHint(), entries, falsePositives, falseRecommendations, dossierId != null);
+                }).sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList());
 
                 dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
 
                 Colors colors = dictionaryClient.getColors(dossierTemplateId);
 
-                dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
-                dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
-                dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
-                dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
+                dictionaryRepresentation.setDefaultColor(convertColor(DEFAULT_COLOR));
+                dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAddColor()));
+                dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemoveColor()));
+                dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getSkippedColor()));
                 dictionaryRepresentation.setDossierTemplateId(dossierTemplateId);
                 dictionaryRepresentation.setDictionaryVersion(version);
                 dictionaryRepresentation.setDictionary(dictionary);
@@ -193,11 +211,11 @@ public class DictionaryService {
             falsePositives.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
             falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
         }
+        log.info("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}", entries.size(), falsePositives.size(), falseRecommendations.size(), type.getType());
         return new DictionaryEntries(entries, falsePositives, falseRecommendations);
     }
 
 
-
     private float[] convertColor(String hex) {
 
         Color color = Color.decode(hex);
@@ -225,6 +243,7 @@ public class DictionaryService {
     }
 
 
+    @Timed("redactmanager_getDeepCopyDictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
         List<DictionaryModel> copy = new ArrayList<>();
@@ -244,7 +263,12 @@ public class DictionaryService {
             dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
         }
 
-        return new Dictionary(copy.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
+        return new Dictionary(copy.stream()
+                .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
+                .collect(Collectors.toList()), DictionaryVersion.builder()
+                .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
+                .dossierVersion(dossierDictionaryVersion)
+                .build());
     }
 
 
@@ -255,10 +279,12 @@ public class DictionaryService {
 
 
     private Long getVersion(DictionaryRepresentation dictionaryRepresentation) {
+
         if (dictionaryRepresentation == null) {
             return null;
         } else {
             return dictionaryRepresentation.getDictionaryVersion();
         }
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
@@ -41,6 +43,7 @@ public class DroolsExecutionService {
     }
 
 
+    @Timed("redactmanager_executeRules")
     public Section executeRules(KieContainer kieContainer, Section section) {
 
         KieSession kieSession = kieContainer.newKieSession();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@@ -13,6 +14,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -61,8 +63,7 @@ public class EntityRedactionService {
         List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
         for (SectionText reanalysisSection : reanalysisSections) {
 
-            Entities entities = findEntities(reanalysisSection.getSearchableText(), reanalysisSection.getHeadline(), reanalysisSection
-                    .getSectionNumber(), dictionary, local, nerEntities, reanalysisSection.getCellStarts());
+            Entities entities = findEntities(reanalysisSection.getSearchableText(), reanalysisSection.getHeadline(), reanalysisSection.getSectionNumber(), dictionary, local, nerEntities, reanalysisSection.getCellStarts(), analyzeRequest.getManualRedactions());
 
             if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
                 surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection
@@ -71,19 +72,26 @@ public class EntityRedactionService {
                 surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
             }
 
-
             if (!local && analyzeRequest.getManualRedactions() != null) {
 
-                var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream()
+                var approvedForceRedactions = analyzeRequest.getManualRedactions()
+                        .getForceRedactions()
+                        .stream()
                         .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
                         .filter(fr -> fr.getRequestDate() != null)
                         .collect(Collectors.toList());
                 // only approved id removals, that haven't been forced back afterwards
-                var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
+                var idsToRemove = analyzeRequest.getManualRedactions()
+                        .getIdsToRemove()
+                        .stream()
                         .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
                         .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
-                        .map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
+                        .filter(idr -> approvedForceRedactions.stream()
+                                .noneMatch(forceRedact -> forceRedact.getAnnotationId()
+                                        .equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
+                                        .isAfter(idr.getRequestDate())))
+                        .map(IdRemoval::getAnnotationId)
+                        .collect(Collectors.toSet());
 
                 if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
                         .isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
@@ -110,13 +118,13 @@ public class EntityRedactionService {
                 }));
             }
 
+            log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
 
             sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                     .isLocal(false)
                     .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream
-                            .concat(entities.getEntities().stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber())
-                                    .stream())
+                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
+                                    .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream())
                             .collect(Collectors.toSet()) : entities.getEntities())
                     .nerEntities(entities.getNerEntities())
                     .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
@@ -127,8 +135,10 @@ public class EntityRedactionService {
                     .searchableText(reanalysisSection.getSearchableText())
                     .dictionary(dictionary)
                     .images(reanalysisSection.getImages())
+                    .sectionAreas(reanalysisSection.getSectionAreas())
                     .fileAttributes(analyzeRequest.getFileAttributes())
-                    .build(), reanalysisSection.getSearchableText()));
+                    .manualRedactions(analyzeRequest.getManualRedactions())
+                    .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
 
         }
 
@@ -136,6 +146,16 @@ public class EntityRedactionService {
         sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
             Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
             EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+
+            var entriesWithoutSurroundingText = analysedSection.getEntities().stream().filter(e -> e.getTextAfter() == null && e.getTextBefore() == null).collect(Collectors.toSet());
+
+            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts()
+                    .isEmpty()) {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary, sectionSearchableTextPair.getCellStarts());
+            } else {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+            }
+
             entities.addAll(analysedSection.getEntities());
 
             if (!local) {
@@ -144,6 +164,7 @@ public class EntityRedactionService {
                 }
                 addLocalValuesToDictionary(analysedSection, dictionary);
             }
+
         });
 
         return entities;
@@ -162,11 +183,7 @@ public class EntityRedactionService {
 
             for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
                 entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
-                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
-                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
-                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
-                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity
-                                .getReferences(), entity.getEntityType()));
+                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity.getReferences(), entity.getEntityType()));
             }
         }
         return entitiesPerPage;
@@ -187,26 +204,25 @@ public class EntityRedactionService {
 
     private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {
 
-        analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> {
-            analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
+        analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
 
-                if (dictionary.getLocalAccessMap().get(key) == null) {
-                    log.warn("Dictionary {} is null", key);
-                }
+            if (dictionary.getLocalAccessMap().get(key) == null) {
+                log.warn("Dictionary {} is null", key);
+            }
 
-                if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
-                    log.warn("Dictionary {} localEntries is null", key);
-                }
+            if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
+                log.warn("Dictionary {} localEntries is null", key);
+            }
 
-                dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
-            });
-        });
+            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
+        }));
     }
 
 
+    @Timed("redactmanager_findEntities")
     private Entities findEntities(SearchableText searchableText, String headline, int sectionNumber,
                                   Dictionary dictionary, boolean local, NerEntities nerEntities,
-                                  List<Integer> cellStarts) {
+                                  List<Integer> cellStarts, ManualRedactions manualRedactions) {
 
         Set<Entity> found = new HashSet<>();
         String searchableString = searchableText.asString();
@@ -219,8 +235,7 @@ public class EntityRedactionService {
         for (DictionaryModel model : dictionary.getDictionaryModels()) {
 
             var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
-            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
-                    searchImplementation, model, new FindEntityDetails(model.getType(),headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, local? EntityType.RECOMMENDATION: EntityType.ENTITY));
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString, searchImplementation, model, new FindEntityDetails(model.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
 
             EntitySearchUtils.addOrAddEngine(found, entities);
         }
@@ -230,7 +245,8 @@ public class EntityRedactionService {
             nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
         }
 
-        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound);
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java

@@ -7,10 +7,12 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
 import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
+import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 
 @Service
@@ -23,6 +25,7 @@ public class ImportedRedactionService {
     private final RedactionStorageService redactionStorageService;
 
 
+    @Timed("redactmanager_processImportedRedactions")
     public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId, String dossierId, String fileId,
                                                              List<RedactionLogEntry> redactionLogEntries,
                                                              boolean addImportedRedactions) {
@@ -68,12 +71,13 @@ public class ImportedRedactionService {
     private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
 
         for (Rectangle rectangle : redactionLogEntry.getPositions()) {
+            var normalizedRectangle = normalize(rectangle);
             if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
                 var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
                 for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
                     for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
-                        if (intersects(importedRedactionPosition, rectangle)) {
-                            if(redactionLogEntry.getImportedRedactionIntersections() == null){
+                        if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
+                            if (redactionLogEntry.getImportedRedactionIntersections() == null) {
                                 redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
                             }
                             redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
@@ -85,15 +89,57 @@ public class ImportedRedactionService {
     }
 
 
-    private boolean intersects(Rectangle importedPosition, Rectangle redactionLogPosition) {
+    boolean valueInRange(float value, float min, float max) {
 
-        return redactionLogPosition.getTopLeft()
-                .getX() + redactionLogPosition.getWidth() > importedPosition.getTopLeft()
-                .getX() && redactionLogPosition.getTopLeft()
-                .getY() + redactionLogPosition.getHeight() > importedPosition.getTopLeft()
-                .getY() && redactionLogPosition.getTopLeft().getX() < importedPosition.getTopLeft()
-                .getX() + importedPosition.getWidth() && redactionLogPosition.getTopLeft()
-                .getY() < importedPosition.getTopLeft().getY() + importedPosition.getHeight();
+        return round(value) >= round(min) && round(value) <= round(max);
+    }
+
+
+    boolean rectOverlap(Rectangle a, Rectangle b) {
+
+        boolean xOverlap = valueInRange(a.getTopLeft().getX(), b.getTopLeft().getX(), b.getTopLeft()
+                .getX() + b.getWidth()) || valueInRange(b.getTopLeft().getX(), a.getTopLeft().getX(), a.getTopLeft()
+                .getX() + a.getWidth());
+
+        boolean yOverlap = valueInRange(a.getTopLeft().getY(), b.getTopLeft().getY(), b.getTopLeft()
+                .getY() + b.getHeight()) || valueInRange(b.getTopLeft().getY(), a.getTopLeft().getY(), a.getTopLeft()
+                .getY() + a.getHeight());
+
+        return xOverlap && yOverlap;
+    }
+
+
+    private Rectangle normalize(Rectangle rectangle) {
+
+        Rectangle r = new Rectangle();
+        Point p = new Point();
+
+        if (rectangle.getWidth() < 0) {
+            p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
+            r.setWidth(Math.abs(rectangle.getWidth()));
+        } else {
+            p.setX(rectangle.getTopLeft().getX());
+            r.setWidth(rectangle.getWidth());
+        }
+
+        if (rectangle.getHeight() < 0) {
+            p.setY(rectangle.getTopLeft().getY() + rectangle.getHeight());
+            r.setHeight(Math.abs(rectangle.getHeight()));
+        } else {
+            p.setY(rectangle.getTopLeft().getY());
+            r.setHeight(rectangle.getHeight());
+        }
+
+        r.setTopLeft(p);
+        r.setPage(rectangle.getPage());
+        return r;
+    }
+
+
+    private float round(float value) {
+
+        double d = Math.pow(10, 0);
+        return (float) (Math.round(value * d) / d);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -1,9 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
@@ -17,14 +23,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
 
 @Slf4j
 @Service
@@ -34,17 +36,16 @@ public class ManualRedactionSurroundingTextService {
     private final RedactionStorageService redactionStorageService;
     private final SurroundingWordsService surroundingWordsService;
 
-
+    @Timed("redactmanager_SurroundingTextAnalysis")
     public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
 
         long startTime = System.currentTimeMillis();
         Text text = redactionStorageService.getText(dossierId, fileId);
         List<ManualRedactionEntry> processedAddRedactions = new ArrayList<>();
-        List<ManualResizeRedaction> processedResizeRedactions = new ArrayList<>();
 
         for (SectionText sectionText : text.getSectionTexts()) {
 
-            if (manualRedactions.getEntriesToAdd().isEmpty() && manualRedactions.getResizeRedactions().isEmpty()) {
+            if (manualRedactions.getEntriesToAdd().isEmpty()) {
                 break;
             }
 
@@ -61,23 +62,10 @@ public class ManualRedactionSurroundingTextService {
                         addItty.remove();
                     }
                 }
-
-                var resizeItty = manualRedactions.getResizeRedactions().iterator();
-                while (resizeItty.hasNext()) {
-                    var manualResizeRedaction = resizeItty.next();
-                    if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
-                        var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
-                        manualResizeRedaction.setTextBefore(surroundingText.getLeft());
-                        manualResizeRedaction.setTextAfter(surroundingText.getRight());
-                        processedResizeRedactions.add(manualResizeRedaction);
-                        resizeItty.remove();
-                    }
-                }
             }
         }
 
         manualRedactions.getEntriesToAdd().addAll(processedAddRedactions);
-        manualRedactions.getResizeRedactions().addAll(processedResizeRedactions);
 
         return AnalyzeResult.builder()
                 .dossierId(dossierId)
@@ -88,11 +76,11 @@ public class ManualRedactionSurroundingTextService {
     }
 
 
-    private Pair<String, String> findSurroundingText(SectionText sectionText, String value, List<Rectangle> toFindPositions) {
+    private Pair<String, String> findSurroundingText(SectionText sectionText, String value,
+                                                     List<Rectangle> toFindPositions) {
 
-        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), new SearchImplementation(value, false),
-                new FindEntityDetails("dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, EntityType.ENTITY));
-        Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null);
+        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), new SearchImplementation(value, false), new FindEntityDetails("dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, EntityType.ENTITY));
+        Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null, null);
 
         Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java

@@ -2,6 +2,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
@@ -18,6 +20,7 @@ public class RedactionChangeLogService {
     private final RedactionStorageService redactionStorageService;
 
 
+    @Timed("redactmanager_computeChanges")
     public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) {
 
         long start = System.currentTimeMillis();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -5,7 +5,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
@@ -13,7 +12,6 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
@@ -22,6 +20,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -33,6 +32,7 @@ public class RedactionLogCreatorService {
     private final DictionaryService dictionaryService;
 
 
+    @Timed("redactmanager_createRedactionLog")
     public List<RedactionLogEntry> createRedactionLog(PageEntities pageEntities, int numberOfPages,
                                                       String dossierTemplateId) {
 
@@ -112,10 +112,7 @@ public class RedactionLogCreatorService {
                 redactionLogEntry.setId(entityPositionSequence.getId());
 
                 if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
-                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
-                            .stream()
-                            .flatMap(seq -> seq.getTextPositions().stream())
-                            .collect(Collectors.toList()), page);
+                    List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences());
 
                     redactionLogEntry.getPositions().addAll(rectanglesPerLine);
 
@@ -132,38 +129,43 @@ public class RedactionLogCreatorService {
     }
 
 
-    private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {
+    private List<Rectangle> getRectanglesPerLine(List<TextPositionSequence> textPositionSequences) {
 
         List<Rectangle> rectangles = new ArrayList<>();
-        if (textPositions.size() == 1) {
-            rectangles.add(TextPositionSequence.fromData(textPositions, page).getRectangle());
-        } else {
-            float x = textPositions.get(0).getXDirAdj();
-            float y = textPositions.get(0).getYDirAdj();
-            float width = textPositions.get(0).getWidthDirAdj();
-            float height = textPositions.get(0).getHeightDir();
-            int startIndex = 0;
 
-            for (int i = 1; i < textPositions.size(); i++) {
-                float xDirAdj = textPositions.get(i).getXDirAdj();
-                float yDirAdj = textPositions.get(i).getYDirAdj();
-                float widthDir = textPositions.get(i).getWidthDirAdj();
-                float heightDir = textPositions.get(i).getHeightDir();
+        if (textPositionSequences.size() == 1) {
+            rectangles.add(textPositionSequences.get(0).getRectangle());
+            return rectangles;
+        }
 
-                if (!(isCharInSameLine(y, yDirAdj, height, heightDir) && isCharClose(x, xDirAdj, width))) {
-                    rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, i), page)
-                            .getRectangle());
-                    y = yDirAdj;
-                    width = widthDir;
-                    height = heightDir;
-                    startIndex = i;
-                }
-                x = xDirAdj;
+        TextPositionSequence combinedSequence = new TextPositionSequence();
+        for (int i = 0; i < textPositionSequences.size(); i++) {
+
+            if(combinedSequence.getTextPositions().isEmpty()){
+                combinedSequence = textPositionSequences.get(i);
+                continue;
             }
-            if (startIndex != textPositions.size()) {
-                rectangles.add(TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page)
-                        .getRectangle());
+
+            float lastSeqX = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getXDirAdj();
+            float lastSeqY = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getYDirAdj();
+            float lastSeqWidth = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getWidthDirAdj();
+            float lastSeqHeight = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getHeightDir();
+
+            float currentSeqX = textPositionSequences.get(i).getTextPositions().get(0).getXDirAdj();
+            float currentSeqY = textPositionSequences.get(i).getTextPositions().get(0).getYDirAdj();
+            float currentSeqHeight = textPositionSequences.get(i).getTextPositions().get(0).getHeightDir();
+
+            if (isCharInSameLine(lastSeqY, currentSeqY, lastSeqHeight, currentSeqHeight) && isCharClose(lastSeqX, currentSeqX, lastSeqWidth)) {
+                combinedSequence.getTextPositions().addAll(textPositionSequences.get(i).getTextPositions());
+            } else {
+                rectangles.add(combinedSequence.getRectangle());
+                combinedSequence = textPositionSequences.get(i);
             }
+
+        }
+
+        if(!combinedSequence.getTextPositions().isEmpty()) {
+            rectangles.add(combinedSequence.getRectangle());
         }
 
         return rectangles;
@@ -205,7 +207,8 @@ public class RedactionLogCreatorService {
                 .redacted(entity.isRedaction())
                 .isHint(isHint(entity.getType(), dossierTemplateId))
                 .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
+                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType()
+                        .equals(EntityType.FALSE_RECOMMENDATION))
                 .section(entity.getHeadline())
                 .sectionNumber(entity.getSectionNumber())
                 .matchedRule(entity.getMatchedRule())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -1,40 +1,65 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import java.awt.Color;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.Change;
+import com.iqser.red.service.redaction.v1.model.ChangeType;
+import com.iqser.red.service.redaction.v1.model.ManualChange;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.awt.*;
-import java.time.OffsetDateTime;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class RedactionLogMergeService {
 
-
+    private static final String DELETED_TYPE_COLOR = "#9398a0";
     private final SectionTextService sectionTextService;
 
     private final RedactionStorageService redactionStorageService;
 
+
+    @Timed("redactmanager_getMergedRedactionLog")
     public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
+
         log.debug("Requested preview for: {}", redactionRequest);
 
         var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
@@ -68,16 +93,16 @@ public class RedactionLogMergeService {
         return merged;
     }
 
-    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
-                                               Set<Integer> excludedPages, List<Type> types, Colors colors) {
+
+    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
+                                               ManualRedactions manualRedactions, Set<Integer> excludedPages,
+                                               List<Type> types, Colors colors) {
 
         var skippedImportedRedactions = new HashSet<>();
-
         log.info("Merging Redaction log with manual redactions");
         if (manualRedactions != null) {
 
-
-            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
+            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types, redactionLog.getAnalysisNumber());
 
             redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
 
@@ -108,9 +133,12 @@ public class RedactionLogMergeService {
 
         Set<String> processedIds = new HashSet<>();
         redactionLog.getRedactionLogEntry().removeIf(entry -> {
+
             if (entry.getImportedRedactionIntersections() != null) {
                 entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) {
+                if (!entry.getImportedRedactionIntersections()
+                        .isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType()
+                        .equals("image") || entry.getType().equals("ocr")))) {
                     return true;
                 }
             }
@@ -175,7 +203,6 @@ public class RedactionLogMergeService {
                 String manualOverrideReason = null;
                 if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
 
-
                     redactionLogEntry.setType(imageRecategorization.getType());
                     redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
 
@@ -192,16 +219,20 @@ public class RedactionLogMergeService {
                     redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
                 }
 
-                redactionLogEntry.setReason(manualOverrideReason);
+                if(manualOverrideReason != null) {
+                    redactionLogEntry.setReason(manualOverrideReason);
+                }
 
-                redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
-                        .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                        .withChange("type", imageRecategorization.getType()));
+                redactionLogEntry.getManualChanges()
+                        .add(ManualChange.from(imageRecategorization)
+                                .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
+                                .withChange("type", imageRecategorization.getType()));
             }
 
             if (mrw.getItem() instanceof IdRemoval) {
                 var manualRemoval = (IdRemoval) mrw.getItem();
-                if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
+                if (manualRemoval.getStatus()
+                        .equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
                     log.debug("Skipping merge for dictionary-modifying entry");
                 } else {
                     String manualOverrideReason = null;
@@ -215,11 +246,13 @@ public class RedactionLogMergeService {
                         redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                     }
 
-                    redactionLogEntry.setReason(manualOverrideReason);
+                    if(manualOverrideReason != null) {
+                        redactionLogEntry.setReason(manualOverrideReason);
+                    }
 
-                    redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
-                            .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
-                                    ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
+                    redactionLogEntry.getManualChanges()
+                            .add(ManualChange.from(manualRemoval)
+                                    .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
                 }
             }
 
@@ -243,7 +276,10 @@ public class RedactionLogMergeService {
                     redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                 }
 
-                redactionLogEntry.setReason(manualOverrideReason);
+                if(manualOverrideReason != null) {
+                    redactionLogEntry.setReason(manualOverrideReason);
+                }
+
                 var manualChange = ManualChange.from(manualForceRedact)
                         .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
 
@@ -269,7 +305,9 @@ public class RedactionLogMergeService {
                     redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                 }
 
-                redactionLogEntry.setReason(manualOverrideReason);
+                if(manualOverrideReason != null) {
+                    redactionLogEntry.setReason(manualOverrideReason);
+                }
 
                 var manualChange = ManualChange.from(manualLegalBasisChange)
                         .withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
@@ -290,21 +328,30 @@ public class RedactionLogMergeService {
                     redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
                     redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                     redactionLogEntry.setValue(manualResizeRedact.getValue());
-                    redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                    redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
+
+                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
+                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
+                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
+                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
+                    }
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
                 } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
                     redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                     redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-                    redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                    redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
+
+                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
+                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
+                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
+                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
+                    }
                 }
 
                 redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
-                        .withManualRedactionType(ManualRedactionType.RESIZE)
-                        .withChange("value", manualResizeRedact.getValue()));
+                redactionLogEntry.getManualChanges()
+                        .add(ManualChange.from(manualResizeRedact)
+                                .withManualRedactionType(ManualRedactionType.RESIZE)
+                                .withChange("value", manualResizeRedact.getValue()));
             }
 
         });
@@ -326,14 +373,15 @@ public class RedactionLogMergeService {
 
 
     public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
-                                                       Map<String, List<Comment>> comments, Colors colors, List<Type> types) {
+                                                       Map<String, List<Comment>> comments, Colors colors,
+                                                       List<Type> types, int analysisNumber) {
 
         List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
 
         for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
 
             if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types);
+                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types, analysisNumber);
                 redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
                 redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
                 redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
@@ -348,9 +396,12 @@ public class RedactionLogMergeService {
         return redactionLogEntries;
     }
 
+
     private List<RedactionLogComment> convert(List<Comment> comments) {
-        return comments == null ? null :comments.stream().map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(),
-                c.getFileId(), c.getDate(), c.getSoftDeletedTime())).collect(Collectors.toList());
+
+        return comments == null ? null : comments.stream()
+                .map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
+                .collect(Collectors.toList());
     }
 
 
@@ -365,13 +416,13 @@ public class RedactionLogMergeService {
 
     @SuppressWarnings("PMD.UselessParentheses")
     private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
-                || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null);
+
+        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
     }
 
 
     private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
-                                                      Colors colors, List<Type> types) {
+                                                      Colors colors, List<Type> types, int analysisNumber) {
 
         var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
 
@@ -396,18 +447,19 @@ public class RedactionLogMergeService {
                 .sectionNumber(-1)
                 .rectangle(manualRedactionEntry.isRectangle())
                 .manualChanges(changeList)
+                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
                 .build();
     }
 
 
-    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction,
-                             boolean skipped, List<Type> types) {
+    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped,
+                             List<Type> types) {
 
         if (requested) {
-            return convertColor(colors.getRequestRemove());
+            return convertColor(colors.getRequestRemoveColor());
         }
         if (skipped || !isRedaction && !isHint(types, type)) {
-            return convertColor(colors.getNotRedacted());
+            return convertColor(colors.getSkippedColor());
         }
         return getColor(types, type);
     }
@@ -416,9 +468,9 @@ public class RedactionLogMergeService {
     private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
 
         if (status.equals(AnnotationStatus.REQUESTED)) {
-            return convertColor(colors.getRequestAdd());
+            return convertColor(colors.getRequestAddColor());
         } else if (status.equals(AnnotationStatus.DECLINED)) {
-            return convertColor(colors.getNotRedacted());
+            return convertColor(colors.getSkippedColor());
         }
         return getColor(types, type);
     }
@@ -432,7 +484,8 @@ public class RedactionLogMergeService {
             return convertColor(foundAndNotDeletedType.get().getHexColor());
         }
         Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0"));
+        return firstDeletedType.map(value -> convertColor(value.getHexColor()))
+                .orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
     }
 
 
@@ -454,11 +507,15 @@ public class RedactionLogMergeService {
         return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
     }
 
+
     private List<Type> getMatchingTypes(List<Type> types, String type) {
+
         return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
     }
 
+
     private boolean isDeletedType(Type type) {
+
         return type.getSoftDeletedTime() != null;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -1,19 +1,35 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.server.classification.model.*;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
+import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
@@ -28,15 +44,10 @@ public class SectionTextBuilderService {
 
             List<Table> tables = paragraph.getTables();
             for (Table table : tables) {
-                if (table.getColCount() == 2) {
-                    sectionTexts.add(processTableAsOneText(table, sectionNumber));
-                } else {
-                    sectionTexts.addAll(processTablePerRow(table, sectionNumber));
-                }
+                sectionTexts.addAll(processTablePerRow(table, sectionNumber));
                 sectionNumber.incrementAndGet();
             }
-            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph
-                    .getImages()));
+            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
             sectionNumber.incrementAndGet();
         }
 
@@ -62,58 +73,50 @@ public class SectionTextBuilderService {
     private List<SectionText> processTablePerRow(Table table, AtomicInteger sectionNumber) {
 
         List<SectionText> sectionTexts = new ArrayList<>();
-        for (List<Cell> row : table.getRows()) {
-            SearchableText searchableRow = new SearchableText();
-            Map<String, CellValue> tabularData = new HashMap<>();
-            int start = 0;
-            List<Integer> cellStarts = new ArrayList<>();
-            SectionText sectionText = new SectionText();
-            for (Cell cell : row) {
+        boolean hasHeader = hasTableHeader(table);
 
+        for (List<Cell> row : table.getRows()) {
+
+            List<TextBlock> textBlocks = new ArrayList<>();
+            List<SectionArea> areas = new ArrayList<>();
+            Map<String, CellValue> tabularData = new HashMap<>();
+            List<Integer> startOffsets = new ArrayList<>();
+
+            int startOffset = 0;
+            for (int cellNum = 0; cellNum < row.size(); cellNum++) {
+
+                Cell cell = row.get(cellNum);
                 if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
                     continue;
                 }
 
-                SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
-                        .getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
-                        .get(0)
-                        .getSequences()
-                        .get(0)
-                        .getPage(), null);
-                sectionText.getSectionAreas().add(sectionArea);
-                sectionText.getTextBlocks().addAll(cell.getTextBlocks());
-
-                int cellStart = start;
+                SectionArea sectionArea = getSectionArea(cell);
+                areas.add(sectionArea);
 
                 if (!cell.isHeaderCell()) {
-                    cell.getHeaderCells().forEach(headerCell -> {
-                        StringBuilder headerBuilder = new StringBuilder();
-                        headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
-                        String headerName = headerBuilder.toString()
-                                .replaceAll("\n", "")
-                                .replaceAll(" ", "")
-                                .replaceAll("-", "");
+                    String headerName = getHeaderName(hasHeader, cell, table.getRows().get(0), row.size(), cellNum);
+                    if (headerName != null) {
                         sectionArea.setHeader(headerName);
-                        tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
-                    });
+                        tabularData.put(headerName, new CellValue(cell.getTextBlocks(), startOffset));
+                    }
                 }
 
-                for (TextBlock textBlock : cell.getTextBlocks()) {
-                    // TODO avoid cell overlap merging.
-                    searchableRow.addAll(textBlock.getSequences());
-                }
-                cellStarts.add(cellStart);
-                start = start + cell.toString().trim().length() + 1;
+                textBlocks.addAll(cell.getTextBlocks());
 
+                startOffsets.add(startOffset);
+                startOffset = startOffset + cell.toString().trim().length() + 1;
             }
 
-            sectionText.setText(searchableRow.toString());
-            sectionText.setHeadline(table.getHeadline());
-            sectionText.setSectionNumber(sectionNumber.intValue());
-            sectionText.setTable(true);
-            sectionText.setTabularData(tabularData);
-            sectionText.setCellStarts(cellStarts);
-            sectionTexts.add(sectionText);
+            sectionTexts.add(SectionText.builder()
+                    .text(getRowText(textBlocks))
+                    .headline(table.getHeadline())
+                    .sectionNumber(sectionNumber.intValue())
+                    .isTable(true)
+                    .tabularData(tabularData)
+                    .cellStarts(startOffsets)
+                    .textBlocks(textBlocks)
+                    .sectionAreas(areas)
+                    .build());
 
             sectionNumber.incrementAndGet();
         }
@@ -122,56 +125,53 @@ public class SectionTextBuilderService {
     }
 
 
-    private SectionText processTableAsOneText(Table table, AtomicInteger sectionNumber) {
+    public String getRowText(List<TextBlock> rowTextBlocks) {
 
-        SearchableText entireTableText = new SearchableText();
-        SectionText sectionText = new SectionText();
-        int start = 0;
-        List<Integer> cellStarts = new ArrayList<>();
-        for (List<Cell> row : table.getRows()) {
-
-
-            for (Cell cell : row) {
-                if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
-                    continue;
-                }
-
-                SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
-                        .getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
-                        .get(0)
-                        .getSequences()
-                        .get(0)
-                        .getPage(), null);
-                sectionText.getTextBlocks().addAll(cell.getTextBlocks());
-                sectionText.getSectionAreas().add(sectionArea);
-
-                for (TextBlock textBlock : cell.getTextBlocks()) {
-                    entireTableText.addAll(textBlock.getSequences());
-                }
-
-
-                cellStarts.add(start);
-                start = start + cell.toString().trim().length() + 1;
-            }
-
-        }
-        sectionText.setCellStarts(cellStarts);
-
-        sectionText.setText(entireTableText.toString());
-        sectionText.setHeadline(table.getHeadline());
-        sectionText.setSectionNumber(sectionNumber.intValue());
-        sectionText.setTable(true);
-        return sectionText;
+        return SearchableText.buildString(rowTextBlocks.stream().map(textBlock -> textBlock.getSequences()).flatMap(List::stream).collect(Collectors.toList()));
     }
 
 
-    private SectionText processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks, String headline,
-                                    AtomicInteger sectionNumber, List<PdfImage> images) {
+    private boolean hasTableHeader(Table table) {
+
+        return table.getRows().stream().anyMatch(row -> row.stream().anyMatch(cell -> !cell.isHeaderCell() && !cell.getHeaderCells().isEmpty()));
+    }
+
+
+    private SectionArea getSectionArea(Cell cell) {
+
+        return new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell.getWidth(), (float) cell.getHeight(),
+                cell.getTextBlocks().get(0).getSequences().get(0).getPage(), null);
+    }
+
+
+    private String getHeaderName(boolean hasHeader, Cell currentCell, List<Cell> firstRow, int currentRowSize, int cellNum) {
+
+        if (hasHeader && !currentCell.getHeaderCells().isEmpty()) {
+            return buildHeaderName(currentCell.getHeaderCells().get(currentCell.getHeaderCells().size() - 1));
+        }
+
+        if (!hasHeader && currentRowSize == firstRow.size() && currentCell != firstRow.get(cellNum)) {
+            return buildHeaderName(firstRow.get(cellNum));
+        }
+
+        return null;
+    }
+
+
+    private String buildHeaderName(Cell cell) {
+
+        StringBuilder headerBuilder = new StringBuilder();
+        cell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
+        return headerBuilder.toString().replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
+    }
+
+
+    private SectionText processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks, String headline, AtomicInteger sectionNumber, List<PdfImage> images) {
 
         SectionText sectionText = new SectionText();
         for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
-            SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock
-                    .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage(), null);
+            SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock.getWidth(),
+                    paragraphTextBlock.getHeight(), paragraphTextBlock.getPage(), null);
             sectionText.getSectionAreas().add(sectionArea);
         }
 
@@ -179,9 +179,7 @@ public class SectionTextBuilderService {
         sectionText.setHeadline(headline);
         sectionText.setSectionNumber(sectionNumber.intValue());
         sectionText.setTable(false);
-        sectionText.setImages(images.stream()
-                .map(image -> convertImage(image, sectionNumber.intValue(), headline))
-                .collect(Collectors.toSet()));
+        sectionText.setImages(images.stream().map(image -> convertImage(image, sectionNumber.intValue(), headline)).collect(Collectors.toSet()));
         sectionText.setTextBlocks(paragraphTextBlocks);
         return sectionText;
     }
@@ -190,9 +188,7 @@ public class SectionTextBuilderService {
     private Image convertImage(PdfImage pdfImage, int sectionNumber, String headline) {
 
         return Image.builder()
-                .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType()
-                        .name()
-                        .toLowerCase(Locale.ROOT))
+                .type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType().name().toLowerCase(Locale.ROOT))
                 .position(pdfImage.getPosition())
                 .sectionNumber(sectionNumber)
                 .section(headline)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java

@@ -4,6 +4,8 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
@@ -19,6 +21,7 @@ public class SurroundingWordsService {
     private final RedactionServiceSettings redactionServiceSettings;
 
 
+    @Timed("redactmanager_addSurroundingText")
     public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
 
         if (entities.isEmpty()) {
@@ -39,6 +42,7 @@ public class SurroundingWordsService {
     }
 
 
+    @Timed("redactmanager_addSurroundingText_tables")
     public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary,
                                    List<Integer> cellstarts) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,12 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
 import java.util.*;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -25,12 +26,10 @@ public class EntitySearchUtils {
 
         if (details.getEntityType() == EntityType.RECOMMENDATION) {
             Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
-            removeFalsePositives(found, falseRecommendations);
-            found.addAll(falseRecommendations);
+            markFalsePositives(found, falseRecommendations);
         } else {
             Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
-            removeFalsePositives(found, falsePositives);
-            found.addAll(falsePositives);
+            markFalsePositives(found, falsePositives);
         }
 
         return found;
@@ -55,7 +54,7 @@ public class EntitySearchUtils {
     }
 
 
-    public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
+    public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary, ManualRedactions manualRedactions) {
 
         Map<String, List<Entity>> entitiesByWord = new HashMap<>();
 
@@ -79,13 +78,45 @@ public class EntitySearchUtils {
             }
         }
 
+        if (manualRedactions != null && manualRedactions.getResizeRedactions() != null && !manualRedactions.getResizeRedactions().isEmpty()) {
+            applyResizeRedactions(entities, manualRedactions);
+        }
+
         removeEntitiesContainedInLarger(entities);
 
         return entities;
     }
 
 
-    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
+    private void applyResizeRedactions(Set<Entity> entitiesWithPositions, ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()) {
+            return;
+        }
+
+        entitiesWithPositions.forEach(e -> e.getPositionSequences()
+                .forEach(pos -> manualRedactions.getResizeRedactions().stream().filter(resize -> resize.getStatus().equals(AnnotationStatus.APPROVED)).forEach(resize -> {
+                    if (resize.getAnnotationId().equals(pos.getId())) {
+                        if (resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
+                            int start = e.getWord().indexOf(resize.getValue());
+                            e.setStart(e.getStart() + start);
+                            e.setEnd(e.getStart() + resize.getValue().length());
+                            e.setResized(true);
+                            e.setWord(resize.getValue());
+                        } else if (resize.getValue().length() > e.getWord().length() && resize.getValue().contains(e.getWord())) {
+                            int start = resize.getValue().indexOf(e.getWord());
+                            e.setStart(e.getStart() - start);
+                            e.setEnd(e.getStart() + resize.getValue().length());
+                            e.setResized(true);
+                            e.setWord(resize.getValue());
+                        }
+                    }
+                })));
+
+    }
+
+
+    public void markFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {
 
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : falsePositives) {
@@ -96,8 +127,12 @@ public class EntitySearchUtils {
                 }
             }
         }
-        wordsToRemove.forEach(entities::remove);
-        entities.removeAll(falsePositives);
+        wordsToRemove.forEach(e -> e.setFalsePositive(true));
+        for (var entity : entities) {
+            if (falsePositives.contains(entity)) {
+                entity.setFalsePositive(true);
+            }
+        }
     }
 
 
@@ -106,13 +141,26 @@ public class EntitySearchUtils {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
+//                // skip cross-type false positives
+//                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
+//                    continue;
+//                }
+//                // skip cross-type false reccommendations
+//                if(word.getEntityType() == EntityType.FALSE_RECOMMENDATION && inner.getEntityType() == EntityType.RECOMMENDATION && !inner.getType().equals(word.getType())){
+//                    continue;
+//                }
+
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                         wordsToRemove.add(word);
-                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                    } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
                             inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
-                        wordsToRemove.add(inner);
+                        if (inner.isResized()) {
+                            wordsToRemove.add(word);
+                        } else {
+                            wordsToRemove.add(inner);
+                        }
                     }
                 }
             }
@@ -166,6 +214,9 @@ public class EntitySearchUtils {
 
             if (existing.getType().equals(found.getType())) {
                 existing.getEngines().addAll(found.getEngines());
+                existing.setLegalBasis(found.getLegalBasis());
+                existing.setMatchedRule(found.getMatchedRule());
+                existing.setRedactionReason(found.getRedactionReason());
                 if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
                         || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
                     existing.setEntityType(EntityType.ENTITY);
@@ -230,6 +281,9 @@ public class EntitySearchUtils {
                 }
                 var existingEntity = existingOptional.get();
                 existingEntity.getEngines().addAll(toAdd.getEngines());
+                existingEntity.setLegalBasis(toAdd.getLegalBasis());
+                existingEntity.setMatchedRule(toAdd.getMatchedRule());
+                existingEntity.setRedactionReason(toAdd.getRedactionReason());
             } else {
                 existing.add(toAdd);
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java

@@ -27,26 +27,48 @@ public class ImageService {
     private final ObjectMapper objectMapper;
     private final RedactionStorageService redactionStorageService;
 
+
     @SneakyThrows
-    public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId){
+    public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId) {
 
         var imageClassificationStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
 
-
         ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class);
 
         Map<Integer, List<PdfImage>> images = new HashMap<>();
         imageServiceResponse.getData().forEach(imageMetadata -> {
-            var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)) : ImageType.OTHER;
-            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
-                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), classification, imageMetadata.getPosition().getPageNumber()));
+            var classification = imageMetadata.getFilters()
+                    .isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification()
+                    .getLabel()
+                    .toUpperCase(Locale.ROOT)) : ImageType.OTHER;
+            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
+                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition()
+                            .getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry()
+                            .getWidth(), imageMetadata.getGeometry()
+                            .getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition()
+                            .getPageNumber()));
+        });
+
+        // Currently This is a copy but, it will be changed later because i don' t think that we should unclassified images.
+        imageServiceResponse.getDataCV().forEach(imageMetadata -> {
+            var classification = imageMetadata.getFilters()
+                    .isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification()
+                    .getLabel()
+                    .toUpperCase(Locale.ROOT)) : ImageType.OTHER;
+            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
+                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition()
+                            .getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry()
+                            .getWidth(), imageMetadata.getGeometry()
+                            .getHeight()), classification, imageMetadata.isAlpha(), imageMetadata.getPosition()
+                            .getPageNumber()));
         });
 
         return images;
     }
 
 
-    public void findOcr(Page page){
+    public void findOcr(Page page) {
+
         page.getImages().forEach(image -> {
             if (image.getImageType().equals(ImageType.OTHER)) {
                 page.getTextBlocks().forEach(textblock -> {
@@ -59,5 +81,4 @@ public class ImageService {
         });
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -1,27 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;
 
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
-import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
-import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.SystemUtils;
-import org.apache.pdfbox.io.MemoryUsageSetting;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -35,6 +13,30 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.pdfbox.io.MemoryUsageSetting;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
+import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -72,7 +74,7 @@ public class PdfSegmentationService {
                 Document document = new Document();
                 List<Page> pages = new ArrayList<>();
 
-                pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(1024 * 1024 * 64));
+                pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(67108864L));
                 pdDocument.setAllSecurityToBeRemoved(true);
                 long pageCount = pdDocument.getNumberOfPages();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -1,16 +1,31 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;
 
-import com.iqser.red.service.redaction.v1.server.classification.model.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.stereotype.Service;
 
-import java.util.*;
-import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 public class SectionsBuilderService {
 
@@ -53,8 +68,7 @@ public class SectionsBuilderService {
                     continue;
                 }
 
-                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
-                        .startsWith("H ") || !document.isHeadlines()) {
+                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {
@@ -100,17 +114,19 @@ public class SectionsBuilderService {
 
     public void addImagesToSections(Document document) {
 
-        Map<Integer, SortedSet<Paragraph>> paragraphMap = new HashMap<>();
+        Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
         for (Paragraph paragraph : document.getParagraphs()) {
             for (AbstractTextContainer container : paragraph.getPageBlocks()) {
-                paragraphMap.computeIfAbsent(container.getPage(), x -> new TreeSet<>()).add(paragraph);
+
+                paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
+
             }
         }
 
         if (paragraphMap.isEmpty()) {
             Paragraph paragraph = new Paragraph();
             document.getParagraphs().add(paragraph);
-            paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
+            paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         // first page is always a paragraph, else we can't process pages 1..N,
@@ -118,12 +134,12 @@ public class SectionsBuilderService {
         if (paragraphMap.get(1) == null) {
             Paragraph paragraph = new Paragraph();
             document.getParagraphs().add(paragraph);
-            paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
+            paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         for (Page page : document.getPages()) {
             for (PdfImage image : page.getImages()) {
-                SortedSet<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
+                List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
                 if (paragraphsOnPage == null) {
                     int i = page.getPageNumber();
                     while (paragraphsOnPage == null) {
@@ -131,27 +147,63 @@ public class SectionsBuilderService {
                         i--;
                     }
                 }
-
-                Float perviousEnd = 0f;
                 for (Paragraph paragraph : paragraphsOnPage) {
-                    Float currentEnd = 0f;
+                    Float xMin = null;
+                    Float yMin = null;
+                    Float xMax = null;
+                    Float yMax = null;
+
                     for (AbstractTextContainer abs : paragraph.getPageBlocks()) {
                         if (abs.getPage() != page.getPageNumber()) {
                             continue;
                         }
-                        if (abs.getMaxY() > currentEnd) {
-                            currentEnd = abs.getMaxY();
+
+                        if (abs.getMinX() < abs.getMaxX()) {
+                            if (xMin == null || abs.getMinX() < xMin) {
+                                xMin = abs.getMinX();
+                            }
+                            if (xMax == null || abs.getMaxX() > xMax) {
+                                xMax = abs.getMaxX();
+                            }
+                        } else {
+                            if (xMin == null || abs.getMaxX() < xMin) {
+                                xMin = abs.getMaxX();
+                            }
+                            if (xMax == null || abs.getMinX() > xMax) {
+                                xMax = abs.getMinX();
+                            }
                         }
+
+                        if (abs.getMinY() < abs.getMaxY()) {
+                            if (yMin == null || abs.getMinY() < yMin) {
+                                yMin = abs.getMinY();
+                            }
+                            if (yMax == null || abs.getMaxY() > yMax) {
+                                yMax = abs.getMaxY();
+                            }
+                        } else {
+                            if (yMin == null || abs.getMaxY() < yMin) {
+                                yMin = abs.getMaxY();
+                            }
+                            if (yMax == null || abs.getMinY() > yMax) {
+                                yMax = abs.getMinY();
+                            }
+                        }
+
                     }
 
-                    if (image.getPosition().getY() >= perviousEnd && image.getPosition().getY() <= currentEnd) {
+                    log.debug("Image position x: {}, y: {}", image.getPosition().getX(), image.getPosition().getY());
+                    log.debug("Paragraph position xMin: {}, xMax: {}, yMin: {}, yMax: {}", xMin, xMax, yMin, yMax);
+
+                    if (xMin != null && xMax != null && yMin != null && yMax != null && image.getPosition().getX() >= xMin && image.getPosition()
+                            .getX() <= xMax && image.getPosition().getY() >= yMin && image.getPosition().getY() <= yMax) {
                         paragraph.getImages().add(image);
                         image.setAppendedToParagraph(true);
                     }
-                    perviousEnd = currentEnd;
                 }
                 if (!image.isAppendedToParagraph()) {
-                    paragraphsOnPage.first().getImages().add(image);
+                    log.debug("Image uses first paragraph");
+                    paragraphsOnPage.get(0).getImages().add(image);
                     image.setAppendedToParagraph(true);
                 }
             }
@@ -166,9 +218,7 @@ public class SectionsBuilderService {
             List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
             List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
             // Allow merging of tables if header row is separated from first logical non-header row
-            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows()
-                    .get(0)
-                    .size() == tableNonHeaderRow.size()) {
+            if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows().get(0).size() == tableNonHeaderRow.size()) {
                 previousTableNonHeaderRow = previousTable.getRows().get(0).stream().map(cell -> {
                     Cell fakeCell = new Cell(cell.getPoints()[0], cell.getPoints()[2]);
                     fakeCell.setHeaderCells(Collections.singletonList(cell));
@@ -178,8 +228,7 @@ public class SectionsBuilderService {
             if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
                 for (int i = currentTable.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
                     List<Cell> row = currentTable.getRows().get(i);
-                    if (row.size() == tableNonHeaderRow.size() && row.stream()
-                            .allMatch(cell -> cell.getHeaderCells().isEmpty())) {
+                    if (row.size() == tableNonHeaderRow.size() && row.stream().allMatch(cell -> cell.getHeaderCells().isEmpty())) {
                         for (int j = 0; j < row.size(); j++) {
                             row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
                         }
@@ -229,24 +278,20 @@ public class SectionsBuilderService {
             TextBlock wordBlock = (TextBlock) container;
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
-                        .getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
-                        .getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
                 alreadyAdded = false;
             } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
                 textBlock.setPage(pageBefore);
                 paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
-                        .getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
-                        .getHeight());
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
             }
             pageBefore = wordBlock.getPage();
             splitByTable = false;
@@ -268,11 +313,7 @@ public class SectionsBuilderService {
 
     private boolean hasInvalidHeaderInformation(Table table) {
 
-        return table.getRows()
-                .stream()
-                .flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
-                .findAny()
-                .isEmpty();
+        return table.getRows().stream().flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells()))).findAny().isEmpty();
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -1,8 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.storage;
 
-import com.dslplatform.json.DslJson;
-import com.dslplatform.json.runtime.Settings;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.InputStream;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
@@ -12,16 +13,12 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
+
+import io.micrometer.core.annotation.Timed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.core.io.InputStreamResource;
-import org.springframework.stereotype.Service;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
 
 @Slf4j
 @Service
@@ -29,10 +26,6 @@ import java.io.InputStream;
 public class RedactionStorageService {
 
     private final StorageService storageService;
-    private final ObjectMapper objectMapper;
-
-    private final DslJson<Object> dslJson = new DslJson<>(Settings.basicSetup());
-
 
     @SneakyThrows
     public InputStream getStoredObject(String storageId) {
@@ -41,21 +34,6 @@ public class RedactionStorageService {
     }
 
 
-    @SneakyThrows
-    public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
-        var baos = new ByteArrayOutputStream();
-        try {
-            dslJson.serialize(any, baos);
-        } catch (com.dslplatform.json.SerializationException e){
-            // Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
-            var bytes = objectMapper.writeValueAsBytes(any);
-            storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), bytes);
-            dslJson.newWriter();
-            return;
-        }
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), baos.toByteArray());
-    }
-
     @SneakyThrows
     public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {
 
@@ -63,86 +41,68 @@ public class RedactionStorageService {
     }
 
 
+    @SneakyThrows
+    @Timed("redactmanager_storeObject")
+    public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
+        storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
+    }
+
+    @Timed("redactmanager_getImportedRedactions")
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Imported redactions not available.");
             return null;
         }
-
-        try {
-            return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not imported redactions", e);
-        }
     }
 
 
+    @Timed("redactmanager_getRedactionLog")
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),RedactionLog.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
         }
 
-        try {
-            return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert RedactionLog", e);
-        }
     }
 
 
+    @Timed("redactmanager_getText")
     public Text getText(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT),Text.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
         }
-
-        try {
-            return dslJson.deserialize(Text.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert Text", e);
-        }
     }
 
 
+    @Timed("redactmanager_getNerEntities")
     public NerEntities getNerEntities(String dossierId, String fileId) {
 
-        InputStreamResource inputStreamResource;
         try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("NER Entities are not available.");
         }
-
-        try {
-            return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert NER Entities", e);
-        }
     }
 
 
+
+    @Timed("redactmanager_getSectionGrid")
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
         try {
-            var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
-            return dslJson.deserialize(SectionGrid.class, sectionGrid.getInputStream());
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID),SectionGrid.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("Section Grid is not available.");
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert RedactionLog", e);
         }
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java

@@ -13,13 +13,20 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public abstract class AbstractTextContainer {
 
+    @JsonIgnore
     protected float minX;
+    @JsonIgnore
     protected float maxX;
+    @JsonIgnore
     protected float minY;
+    @JsonIgnore
     protected float maxY;
+    @JsonIgnore
     protected String classification;
+    @JsonIgnore
     protected int page;
 
+    @JsonIgnore
     private Orientation orientation = Orientation.NONE;
 
     public abstract String getText();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -62,7 +62,7 @@ public class DictionaryServiceTest {
 
         when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
         when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", "#cccccc",
-                "#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc",
+                "#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc","#cccccc",
                 "#cccccc", "#cccccc", "#cccccc", "#cccccc"));
 
         var type1 = new Type();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -1,20 +1,26 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.S3StorageService;
-import com.iqser.red.storage.commons.service.StorageService;
-import lombok.SneakyThrows;
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
-public class FileSystemBackedStorageService implements StorageService{
+import org.apache.commons.io.IOUtils;
+import org.springframework.core.io.InputStreamResource;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+public class FileSystemBackedStorageService implements StorageService {
 
     private final Map<String, File> dataMap = new HashMap<>();
 
@@ -48,36 +54,51 @@ public class FileSystemBackedStorageService implements StorageService{
 
     }
 
-    public List<String> listPaths(){
+    @Override
+    @SneakyThrows
+    public <T> void storeJSONObject(String objectId, T any) {
+        File tempFile = File.createTempFile("test", ".tmp");
+        getMapper().writeValue(new FileOutputStream(tempFile), any);
+        dataMap.put(objectId, tempFile);
+    }
+
+    private ObjectMapper getMapper() {
+
+        return ObjectMapperFactory.create();
+    }
+
+    @Override
+    @SneakyThrows
+    public <T> T readJSONObject(String objectId, Class<T> clazz) {
+        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
+            throw new StorageObjectDoesNotExist("Stored object not found");
+        }
+        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
+    }
+
+    public List<String> listPaths() {
         return new ArrayList<>(dataMap.keySet());
     }
 
 
-    public List<String> listFilePaths(){
+    public List<String> listFilePaths() {
         return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
     }
 
 
-    @SneakyThrows
-    @Override
-    public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
-
-        IOUtils.write(data, new FileOutputStream(tempFile));
-
-        dataMap.put(objectId, tempFile);
-    }
-
     @Override
     @SneakyThrows
     public void storeObject(String objectId, InputStream stream) {
         File tempFile = File.createTempFile("test", ".tmp");
 
-        IOUtils.copy(stream, new FileOutputStream(tempFile));
+        try (var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.copy(stream, fileOutputStream);
+        }
 
         dataMap.put(objectId, tempFile);
     }
 
+
     public void clearStorage() {
         this.dataMap.forEach((k, v) -> {
             v.delete();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server;
 
 import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
@@ -30,7 +31,6 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import lombok.SneakyThrows;
-import org.apache.commons.io.IOUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -41,6 +41,7 @@ import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
+import org.mockito.stubbing.Answer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -58,6 +59,7 @@ import java.io.*;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -89,6 +91,7 @@ public class RedactionIntegrationTest {
     private static final String DOSSIER_REDACTIONS = "dossier_redactions";
     private static final String IMPORTED_REDACTION = "imported_redaction";
     private static final String PII = "PII";
+    private static final String ROTATE_SIMPLE = "RotateSimple";
 
     @Autowired
     private RedactionController redactionController;
@@ -214,30 +217,32 @@ public class RedactionIntegrationTest {
     }
 
 
-    private void mockDictionaryCalls(Long version){
+    private void mockDictionaryCalls(Long version) {
 
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
-        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
-        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
-        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false));
-        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMAGE, false));
-        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(OCR, false));
-        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(LOGO, false));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SIGNATURE, false));
-        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION, false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD, false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE, false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS, true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION, true));
 
     }
 
+
     @Test
     public void test270Rotated() {
 
@@ -325,6 +330,36 @@ public class RedactionIntegrationTest {
     }
 
 
+
+    @Test
+    public void titleExtraction() throws IOException {
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/32 - Emamectin Benzoate Technical - Acute Oral Toxicity - Mouse.pdf");
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
+
+
     @Test
     @Ignore
     @SneakyThrows
@@ -491,14 +526,12 @@ public class RedactionIntegrationTest {
         deleted.add("David Chubb");
         deleted.add("mouse");
 
-
         reanlysisVersions.put("mouse", 3L);
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
 
         when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
 
-
         start = System.currentTimeMillis();
 
         ManualRedactions manualRedactions = new ManualRedactions();
@@ -601,8 +634,7 @@ public class RedactionIntegrationTest {
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
         ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
-        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
 
         long start = System.currentTimeMillis();
         ClassPathResource pdfFileResource = new ClassPathResource(fileName);
@@ -669,7 +701,6 @@ public class RedactionIntegrationTest {
 
         when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
 
-
         start = System.currentTimeMillis();
 
         ManualRedactions manualRedactions = new ManualRedactions();
@@ -715,6 +746,71 @@ public class RedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testRemovePublishedInformations() throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
+        var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
+
+        ClassPathResource typeResource = new ClassPathResource("colors/types.json");
+        TypeReference<List<Type>> typeRefForTypes = new TypeReference<>() {
+        };
+        List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
+
+        AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        ManualRedactions manualRedactions = new ManualRedactions();
+        manualRedactions.getIdsToRemove()
+                .add(IdRemoval.builder()
+                        .annotationId("308dab9015bfafd911568cffe0a7f7de")
+                        .fileId(TEST_FILE_ID)
+                        .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
+                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
+                        .build());
+
+        manualRedactions.getForceRedactions()
+                .add(ManualForceRedaction.builder()
+                        .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                        .fileId(TEST_FILE_ID)
+                        .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
+                        .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
+                        .build());
+
+        manualRedactions.getIdsToRemove()
+                .add(IdRemoval.builder()
+                        .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                        .fileId(TEST_FILE_ID)
+                        .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
+                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
+                        .build());
+
+        request.setManualRedactions(manualRedactions);
+
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .manualRedactions(manualRedactions)
+                .colors(colors)
+                .types(types)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
     @Test
     public void testTableRedaction() throws IOException {
 
@@ -740,6 +836,170 @@ public class RedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testUnicodeProblem() throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/new/unicodeProblem.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
+    @Test
+    public void testRotations() throws IOException {
+
+        System.out.println("testTableRedaction");
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+    @Test
+    public void testRotationsSimple() throws IOException {
+        loadOnlyDictionaryForSimpleFile();
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        System.out.println("testTableRedaction");
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/new/RotateTestFileSimple.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/AnnotatedSimple.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
+    @Test
+    public void testTableHeader() throws IOException {
+
+        System.out.println("testTableHeader");
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/Minimal Examples/NoHeaderTable.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(5);
+
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
+
+    @Test
+    public void testFindDictionaryEntryInResizedEntryPosition() throws IOException {
+
+        System.out.println("testResizeDictFound");
+
+        ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
+        var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
+
+        ClassPathResource typeResource = new ClassPathResource("colors/types.json");
+        TypeReference<List<Type>> typeRefForTypes = new TypeReference<>() {
+        };
+        List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
+
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/new/S157.pdf");
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+
+        manualRedactions.getResizeRedactions()
+                .add(ManualResizeRedaction.builder()
+                        .annotationId("ca2b437e2480a4b5966cb8386020d454")
+                        .fileId(TEST_FILE_ID)
+                        .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
+                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
+                        .value("Bera P")
+                        .positions(List.of(new Rectangle(382.12485f, 235.94246f, 8.768621f, -21.632504f, 1)))
+                        .textBefore("")
+                        .textAfter("")
+                        .build());
+
+        request.setManualRedactions(manualRedactions);
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .manualRedactions(manualRedactions)
+                .colors(colors)
+                .types(types)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
     @Test
     public void testManualRedaction() throws IOException {
 
@@ -834,7 +1094,7 @@ public class RedactionIntegrationTest {
     public void classificationTest() throws IOException {
 
         System.out.println("classificationTest");
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
 
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
 
@@ -1304,6 +1564,15 @@ public class RedactionIntegrationTest {
 
     }
 
+    private void loadOnlyDictionaryForSimpleFile() {
+        dictionary.clear();
+        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+    }
+
 
     private static String loadFromClassPath(String path) {
 
@@ -1361,7 +1630,8 @@ public class RedactionIntegrationTest {
         typeColorMap.put(LOGO, "#ffe187");
         typeColorMap.put(FORMULA, "#ffe187");
         typeColorMap.put(SIGNATURE, "#ffe187");
-        typeColorMap.put(IMPORTED_REDACTION, "#32a852");
+        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
+        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
 
         hintTypeMap.put(VERTEBRATE, true);
         hintTypeMap.put(ADDRESS, false);
@@ -1382,6 +1652,7 @@ public class RedactionIntegrationTest {
         hintTypeMap.put(SIGNATURE, false);
         hintTypeMap.put(DOSSIER_REDACTIONS, false);
         hintTypeMap.put(IMPORTED_REDACTION, false);
+        hintTypeMap.put(ROTATE_SIMPLE, false);
 
         caseInSensitiveMap.put(VERTEBRATE, true);
         caseInSensitiveMap.put(ADDRESS, false);
@@ -1402,6 +1673,7 @@ public class RedactionIntegrationTest {
         caseInSensitiveMap.put(FORMULA, true);
         caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
         caseInSensitiveMap.put(IMPORTED_REDACTION, false);
+        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
 
         recommendationTypeMap.put(VERTEBRATE, false);
         recommendationTypeMap.put(ADDRESS, false);
@@ -1422,6 +1694,7 @@ public class RedactionIntegrationTest {
         recommendationTypeMap.put(LOGO, false);
         recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
         recommendationTypeMap.put(IMPORTED_REDACTION, false);
+        recommendationTypeMap.put(ROTATE_SIMPLE, false);
 
         rankTypeMap.put(PURITY, 155);
         rankTypeMap.put(PII, 150);
@@ -1442,11 +1715,11 @@ public class RedactionIntegrationTest {
         rankTypeMap.put(FORMULA, 26);
         rankTypeMap.put(DOSSIER_REDACTIONS, 200);
         rankTypeMap.put(IMPORTED_REDACTION, 200);
+        rankTypeMap.put(ROTATE_SIMPLE, 150);
 
-        colors.setDefaultColor("#acfc00");
-        colors.setNotRedacted("#cccccc");
-        colors.setRequestAdd("#04b093");
-        colors.setRequestRemove("#04b093");
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
     }
 
 
@@ -1454,8 +1727,7 @@ public class RedactionIntegrationTest {
     private void loadNerForTest() {
 
         ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
     }
 
 
@@ -1493,7 +1765,6 @@ public class RedactionIntegrationTest {
                 .build();
     }
 
-
     private String cleanDictionaryEntry(String entry) {
 
         return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
@@ -1501,6 +1772,8 @@ public class RedactionIntegrationTest {
 
 
     private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+        if (entries == null)
+            entries = Collections.emptyList();
 
         List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
         entries.forEach(entry -> {
@@ -1518,11 +1791,11 @@ public class RedactionIntegrationTest {
     public void testImportedRedactions() throws IOException {
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
-        ClassPathResource pdfFileResource = new ClassPathResource("files/ImportedRedactions/ImportedRedactions.pdf");
-        ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/ImportedRedactions.json");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
+        ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
 
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream()));
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1534,20 +1807,33 @@ public class RedactionIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .build());
 
+        redactionLog.getRedactionLogEntry().forEach(entry -> {
+            if (entry.getValue() == null) {
+                return;
+            }
+            if (entry.getValue().equals("David")) {
+                assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
+            }
+            if (entry.getValue().equals("annotation")) {
+                assertThat(entry.getImportedRedactionIntersections()).isEmpty();
+            }
+        });
+
         try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
     }
 
+
     @Test
     public void testExpandByPrefixRegEx() throws IOException {
 
-        assertThat(dictionary.get(AUTHOR).contains("Robinson"));
-        assertThat(! dictionary.get(AUTHOR).contains("Mrs. Robinson"));
-        assertThat(dictionary.get(AUTHOR).contains("Bojangles"));
-        assertThat(! dictionary.get(AUTHOR).contains("Mr. Bojangles"));
-        assertThat(dictionary.get(AUTHOR).contains("Tambourine Man"));
-        assertThat(! dictionary.get(AUTHOR).contains("Mr. Tambourine Man"));
+        assertThat(dictionary.get(AUTHOR)).contains("Robinson");
+        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mrs. Robinson");
+        assertThat(dictionary.get(AUTHOR)).contains("Bojangles");
+        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Bojangles");
+        assertThat(dictionary.get(AUTHOR)).contains("Tambourine Man");
+        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Tambourine Man");
 
         String fileName = "files/mr-mrs.pdf";
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
@@ -1578,6 +1864,7 @@ public class RedactionIntegrationTest {
         assertThat(values).contains("Mr. Tambourine Man");
     }
 
+
     @SneakyThrows
     private AnalyzeRequest prepareStorage(InputStream stream) {
 
@@ -1588,9 +1875,7 @@ public class RedactionIntegrationTest {
                 .lastProcessed(OffsetDateTime.now())
                 .build();
 
-        var bytes = IOUtils.toByteArray(stream);
-
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
 
         return request;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -0,0 +1,741 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static java.util.Map.entry;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.annotation.IfProfileValue;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.Change;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.model.ManualChange;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RulesTest.RulesTestConfiguration.class)
+public class RulesTest {
+
+    private static final String RULES_PATH = "drools/testRules.drl";
+    private static final String RULES = loadFromClassPath(RULES_PATH);
+    private static final String VERTEBRATE = "vertebrate";
+    private static final String ADDRESS = "CBI_address";
+    private static final String AUTHOR = "CBI_author";
+    private static final String SPONSOR = "CBI_sponsor";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
+    private static final String PUBLISHED_INFORMATION = "published_information";
+    private static final String TEST_METHOD = "test_method";
+    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";
+    private static final String LOGO = "logo";
+    private static final String SIGNATURE = "signature";
+    private static final String FORMULA = "formula";
+    private static final String OCR = "ocr";
+    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
+    private static final String IMPORTED_REDACTION = "imported_redaction";
+    private static final String PII = "PII";
+    private static final String RESOURCES_PATH = "src/test/resources/";
+    private static final String REDACTION_LOG_PATH = "RedactionLog/";
+    private static final String REDACTION_LOG_RESOURCES_PATH = RESOURCES_PATH + REDACTION_LOG_PATH;
+    private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    private final static String TEST_DOSSIER_ID = "123";
+    private String TEST_FILE_ID = "123";
+    private int counter = 0;
+    private int fileSize = 0;
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
+    private final Map<String, List<String>> falsePositive = new HashMap<>();
+    private static final Map<String, String> typeColorMap = Map.ofEntries(entry(VERTEBRATE, "#ff85f7"), entry(ADDRESS, "#ffe187"), entry(AUTHOR, "#ffe187"), entry(SPONSOR, "#85ebff"), entry(NO_REDACTION_INDICATOR, "#be85ff"), entry(REDACTION_INDICATOR, "#caff85"), entry(HINT_ONLY, "#abc0c4"), entry(MUST_REDACT, "#fab4c0"), entry(PUBLISHED_INFORMATION, "#85ebff"), entry(TEST_METHOD, "#91fae8"), entry(PII, "#66ccff"), entry(PURITY, "#ffe187"), entry(IMAGE, "#fcc5fb"), entry(OCR, "#fcc5fb"), entry(LOGO, "#ffe187"), entry(FORMULA, "#ffe187"), entry(SIGNATURE, "#ffe187"), entry(IMPORTED_REDACTION, "#fcfbe6"));
+    private static final Map<String, Boolean> hintTypeMap = Map.ofEntries(entry(VERTEBRATE, true), entry(ADDRESS, false), entry(AUTHOR, false), entry(SPONSOR, false), entry(NO_REDACTION_INDICATOR, true), entry(REDACTION_INDICATOR, true), entry(HINT_ONLY, true), entry(MUST_REDACT, true), entry(PUBLISHED_INFORMATION, true), entry(TEST_METHOD, true), entry(PII, false), entry(PURITY, false), entry(IMAGE, true), entry(OCR, true), entry(FORMULA, false), entry(LOGO, false), entry(SIGNATURE, false), entry(DOSSIER_REDACTIONS, false), entry(IMPORTED_REDACTION, false));
+    private static final Map<String, Boolean> caseInSensitiveMap = Map.ofEntries(entry(VERTEBRATE, true), entry(ADDRESS, false), entry(AUTHOR, false), entry(SPONSOR, false), entry(NO_REDACTION_INDICATOR, true), entry(REDACTION_INDICATOR, true), entry(HINT_ONLY, true), entry(MUST_REDACT, true), entry(PUBLISHED_INFORMATION, true), entry(TEST_METHOD, false), entry(PII, false), entry(PURITY, false), entry(IMAGE, true), entry(OCR, true), entry(SIGNATURE, true), entry(LOGO, true), entry(FORMULA, true), entry(DOSSIER_REDACTIONS, false), entry(IMPORTED_REDACTION, false));
+    private static final Map<String, Boolean> recommendationTypeMap = Map.ofEntries(entry(VERTEBRATE, false), entry(ADDRESS, false), entry(AUTHOR, false), entry(SPONSOR, false), entry(NO_REDACTION_INDICATOR, false), entry(REDACTION_INDICATOR, false), entry(HINT_ONLY, false), entry(MUST_REDACT, false), entry(PUBLISHED_INFORMATION, false), entry(TEST_METHOD, false), entry(PII, false), entry(PURITY, false), entry(IMAGE, false), entry(OCR, false), entry(FORMULA, false), entry(SIGNATURE, false), entry(LOGO, false), entry(DOSSIER_REDACTIONS, false), entry(IMPORTED_REDACTION, false));
+    private static final Map<String, Integer> rankTypeMap = Map.ofEntries(entry(PURITY, 155), entry(PII, 150), entry(ADDRESS, 140), entry(AUTHOR, 130), entry(SPONSOR, 120), entry(VERTEBRATE, 110), entry(MUST_REDACT, 100), entry(REDACTION_INDICATOR, 90), entry(NO_REDACTION_INDICATOR, 80), entry(PUBLISHED_INFORMATION, 70), entry(TEST_METHOD, 60), entry(HINT_ONLY, 50), entry(IMAGE, 30), entry(OCR, 29), entry(LOGO, 28), entry(SIGNATURE, 27), entry(FORMULA, 26), entry(DOSSIER_REDACTIONS, 200), entry(IMPORTED_REDACTION, 200));
+    private final Colors colors = new Colors();
+
+    @Autowired
+    private RedactionController redactionController;
+    @Autowired
+    private AnnotationService annotationService;
+    @Autowired
+    private AnalyzeService analyzeService;
+    @Autowired
+    private ObjectMapper objectMapper;
+    @MockBean
+    private RulesClient rulesClient;
+    @MockBean
+    private DictionaryClient dictionaryClient;
+    @Autowired
+    private RedactionStorageService redactionStorageService;
+    @Autowired
+    private StorageService storageService;
+    @Autowired
+    private ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
+    @MockBean
+    private AmazonS3 amazonS3;
+    @MockBean
+    private RabbitTemplate rabbitTemplate;
+    @MockBean
+    private LegalBasisClient legalBasisClient;
+
+
+    @Before
+    public void stubClients() {
+
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .build()));
+
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    @After
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    /**
+     * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
+     * If the RedactionLog already exists, it will be overwritten
+     * Test is ignored, because it's for manual tests.
+     */
+    @Ignore
+    @Test
+    public void generateRedactionLogForOneFile() {
+
+        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
+        fileSize = 1;
+        generateAndSaveRedactionLog(fileName, true);
+    }
+
+
+    /**
+     * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
+     * If a RedactionLog already exists, the generating for this will be skipped
+     * Please commit generated files, if not the following tests will fail
+     */
+    @Test
+    public void generateRedactionLogForAllFiles() {
+
+        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
+        System.out.println("Will generate RedactionLog for " + files.size() + " files.");
+        fileSize = files.size();
+        files.forEach(this::generateAndSaveRedactionLog);
+    }
+
+
+    /**
+     * Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
+     * If RedactionLog Json does not exist, test will fail.
+     */
+    @Test
+    public void analyseFileAndCompareRedactionLog() {
+
+        String fileName = "files/RulesTest/SYNGENTA_EFSA_sanitisation_GFL_v1_withHighlights.pdf";
+        fileSize = 1;
+        analyseFileAndCompareRedactionLog(fileName);
+    }
+
+
+    /**
+     * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
+     * If RedactionLog Json for one file does not exist, whole test will fail.
+     */
+    @IfProfileValue(name = "test-groups", value = "rules-test")
+    @Test
+    public void analyseAllFilesAndCompareRedactionLogs() {
+
+        Set<String> fileNames = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
+        System.out.println("Will analyse " + fileNames.size() + " files and compare its RedactionLogs.");
+        fileSize = fileNames.size();
+
+        Map<String, AssertionError> failedFiles = new HashMap<>();
+
+        for (String fileName : fileNames) {
+
+            try {
+                analyseFileAndCompareRedactionLog(fileName);
+            } catch (AssertionError assertionError) {
+                failedFiles.put(fileName, assertionError);
+            }
+        }
+
+        if (!failedFiles.isEmpty()) {
+            log.warn("WARNING: {} files from {} failed", failedFiles.size(), fileSize);
+            for (String fileName : failedFiles.keySet()) {
+                log.warn(" - '{}' failed with Error: {} See line {} in {}", fileName, failedFiles.get(fileName), failedFiles.get(fileName)
+                        .getStackTrace()[0].getLineNumber(), failedFiles.get(fileName).getStackTrace()[0].getClassName());
+            }
+        }
+
+        assertThat(failedFiles).isEmpty();
+
+    }
+
+
+    private void generateAndSaveRedactionLog(String fileName) {
+
+        generateAndSaveRedactionLog(fileName, false);
+    }
+
+
+    @SneakyThrows
+    private void generateAndSaveRedactionLog(String fileName, boolean force) {
+
+        counter++;
+        if (!force && doesRedactionLogExist(fileName)) {
+            System.out.println("(" + counter + "/" + fileSize + ") Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
+
+        } else {
+            generateTestFileId(fileName);
+
+            System.out.println("(" + counter + "/" + fileSize + ") Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+
+            loadNerForTest();
+
+            AnalyzeRequest request = prepareStorage(fileName);
+            analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+            analyzeService.analyze(request);
+
+            RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+            saveRedactionLogAsJson(redactionLog, fileName);
+
+        }
+    }
+
+
+    @SneakyThrows
+    public void analyseFileAndCompareRedactionLog(String fileName) {
+
+        counter++;
+        generateTestFileId(fileName);
+        System.out.println("(" + counter + "/" + fileSize + ") Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
+
+        RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
+
+        loadNerForTest();
+
+        AnalyzeRequest request = prepareStorage(fileName);
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        // All timestamps are ignored, because they are for sure different
+        assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
+        assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
+        assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
+        assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
+        assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
+        assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
+
+        assertThat(redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(r -> !r.isFalsePositive())
+                .collect(Collectors.toSet())
+                .size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().stream().filter(r -> !r.isFalsePositive()).collect(Collectors.toSet()).size());
+        assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
+
+        for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
+            var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
+                    .stream()
+                    .filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
+                    .filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
+                    .filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
+                    .findFirst();
+            assertThat(savedRedactionLegalBasis).isPresent();
+        }
+
+        for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
+            var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
+            assertThat(savedRedactionLogEntry).isPresent();
+            assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
+            assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
+            assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
+            assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
+            assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
+            assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
+            assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
+            assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
+            assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
+            assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
+            assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
+            assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
+            assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
+            assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
+            assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
+            assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
+            assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
+            assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
+            assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
+            assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
+            assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
+            assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
+            assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
+            assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
+            assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
+
+            for (Rectangle rectangle : redactionLogEntry.getPositions()) {
+                var savedRectangle = savedRedactionLogEntry.get()
+                        .getPositions()
+                        .stream()
+                        .filter(r -> r.getPage() == rectangle.getPage())
+                        .filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
+                        .filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
+                        .filter(r -> r.getHeight() == rectangle.getHeight())
+                        .filter(r -> r.getWidth() == rectangle.getWidth())
+                        .findFirst();
+                assertThat(savedRectangle).isPresent();
+            }
+
+            for (RedactionLogComment comment : redactionLogEntry.getComments()) {
+                var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
+                assertThat(savedComment).isPresent();
+                assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
+                assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
+                assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
+                assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
+                assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
+
+            }
+
+            for (Change change : redactionLogEntry.getChanges()) {
+                var savedChange = savedRedactionLogEntry.get()
+                        .getChanges()
+                        .stream()
+                        .filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
+                        .filter(c -> c.getType() == change.getType())
+                        .findFirst();
+                assertThat(savedChange).isPresent();
+            }
+
+            for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
+                var savedManualChange = savedRedactionLogEntry.get()
+                        .getManualChanges()
+                        .stream()
+                        .filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
+                        .filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
+                        .filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
+                        .filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
+                        .findFirst();
+                assertThat(savedManualChange).isPresent();
+            }
+
+            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactlyInAnyOrder(redactionLogEntry.getEngines().toArray(new Engine[0]));
+
+            assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
+            assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
+        }
+
+    }
+
+
+    private boolean doesRedactionLogExist(String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+        return file.exists();
+
+    }
+
+
+    @SneakyThrows
+    private void generateTestFileId(String fileName) {
+
+        MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
+        messageDigest.update(fileName.getBytes());
+        TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
+    }
+
+
+    @SneakyThrows
+    private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+
+        String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
+        File dr = new File(directory);
+        boolean created = dr.mkdirs();
+        if (created) {
+            System.out.println("Directory was created");
+        }
+
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+
+        objectMapper.writeValue(file, redactionLog);
+
+        System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
+    }
+
+
+    @SneakyThrows
+    private RedactionLog loadSavedRedactionLog(String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+        String directory = cleanPath(pdfFile.getParentFile().getPath());
+        if (StringUtils.contains(directory, RESOURCES_PATH)) {
+            if (!StringUtils.endsWith(directory, "/")) {
+                directory = directory + "/";
+            }
+            directory = directory + REDACTION_LOG_PATH;
+        } else {
+            if (StringUtils.startsWith(directory, "/")) {
+                directory = StringUtils.substring(directory, 1);
+            }
+            directory = REDACTION_LOG_RESOURCES_PATH + directory;
+        }
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+
+        File file = new File(directory, fileName);
+        assertThat(file).exists();
+
+        return objectMapper.readValue(file, RedactionLog.class);
+    }
+
+
+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = cleanPath(path.toAbsolutePath().toString());
+                    int pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + RESOURCES_PATH.length();
+
+                    fileNames.add(StringUtils.substring(absolutePath, pos));
+                }
+            }
+        }
+        return fileNames;
+    }
+
+
+    private String cleanPath(String path) {
+
+        return StringUtils.replace(path, "\\", "/");
+    }
+
+
+    @SneakyThrows
+    private void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+    }
+
+
+    @SneakyThrows
+    private AnalyzeRequest prepareStorage(String file) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+
+        return prepareStorage(pdfFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    private AnalyzeRequest prepareStorage(InputStream stream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
+
+        return request;
+
+    }
+
+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
+
+        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+    }
+
+
+    private void loadTypeForTest() {
+
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
+    }
+
+
+    private List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+
+    private void mockDictionaryCalls(Long version) {
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
+
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
+                .falseRecommendationEntries(new ArrayList<>())
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(0L).deleted(false).build()));
+        return dictionaryEntries;
+    }
+
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class RulesTestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write(RESOURCES_PATH + RULES_PATH, kieServices.getResources().newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.annotate;
 
+import java.util.List;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -17,4 +21,6 @@ public class AnnotateRequest {
     private String dossierTemplateId;
     private String fileId;
     private ManualRedactions manualRedactions;
+    private Colors colors;
+    private List<Type> types;
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -1,14 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.annotate;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.*;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import lombok.RequiredArgsConstructor;
+import java.awt.Color;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -22,12 +23,21 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;
 
-import java.awt.*;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.CellRectangle;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.SectionRectangle;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;
 
 @Service
 @RequiredArgsConstructor
@@ -46,6 +56,8 @@ public class AnnotationService {
                 .manualRedactions(annotateRequest.getManualRedactions())
                 .dossierId(annotateRequest.getDossierId())
                 .dossierTemplateId(annotateRequest.getDossierTemplateId())
+                        .colors(annotateRequest.getColors())
+                        .types(annotateRequest.getTypes())
                 .build());
         var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
 
@@ -116,7 +128,7 @@ public class AnnotationService {
         annotation.setRectangle(pdRectangle);
         annotation.setQuadPoints(toQuadPoints(rectangles, mediaBox, cropBox));
         if (!redactionLogEntry.isHint()) {
-            annotation.setContents(createAnnotationContent(redactionLogEntry));
+            annotation.setContents(redactionLogEntry.getValue() + " " +createAnnotationContent(redactionLogEntry));
         }
         annotation.setTitlePopup(redactionLogEntry.getId());
         annotation.setAnnotationName(redactionLogEntry.getId());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequenceTest.java

@@ -0,0 +1,71 @@
+package com.iqser.red.service.redaction.v1.server.parsing.model;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.pdfbox.util.Matrix;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.storage.commons.properties.StorageProperties;
+import com.iqser.red.storage.commons.service.ObjectSerializer;
+
+import lombok.SneakyThrows;
+
+public class TextPositionSequenceTest {
+
+    private static final String TEXT_POSITION_SEQUENCE_AS_JSON = "{\n" //
+            + "  \"page\": 1,\n" //
+            + "  \"textPositions\": [],\n" //
+            + "  \"dir\": 180.0,\n" //
+            + "  \"rotation\": 0,\n" //
+            + "  \"pageHeight\": 800,\n" //
+            + "  \"pageWidth\": 600\n"  //
+            + "}";
+
+    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties());
+
+
+    @Test
+    @SneakyThrows
+    public void testDeserializationWithJackson() {
+
+        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(
+                new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+                TextPositionSequence.class);
+
+        assertPropertiesAfterJsonDeserialization(textPositionSequence);
+    }
+
+
+    private void assertPropertiesAfterJsonDeserialization(TextPositionSequence textPositionSequence) {
+
+        assertThat(textPositionSequence.getPage()).isEqualTo(1);
+        assertThat(textPositionSequence.getTextPositions()).hasSize(0);
+        assertThat(textPositionSequence.getDir()).isEqualTo(TextDirection.HALF_CIRCLE);
+        assertThat(textPositionSequence.getRotation()).isEqualTo(0);
+        assertThat(textPositionSequence.getPageHeight()).isEqualTo(800f);
+        assertThat(textPositionSequence.getPageWidth()).isEqualTo(600f);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testDeserializationWithDslJson() {
+
+        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithDslJson(
+                new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+                TextPositionSequence.class);
+
+        assertPropertiesAfterJsonDeserialization(textPositionSequence);
+    }
+
+
+    private Matrix createIdentityMatrix() {
+
+        return new Matrix();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java

@@ -5,9 +5,11 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.MessageType;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
 import java.time.OffsetDateTime;
@@ -19,6 +21,9 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
 
     public static final String FILE_NAME = "test-file";
 
+    @Autowired
+    private AnnotationService annotationService;
+
     @Test
     @SneakyThrows
     public void testFile() {
@@ -81,7 +86,19 @@ public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest
         redactionMessageReceiver.receiveAnalyzeRequest(in, false);
 
 
-        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        var redactionLog = redactionStorageService.getRedactionLog("dossierId","fileId");
+//        redactionLog.getRedactionLogEntry().forEach(entry ->{
+//            log.info("Entry {}", entry);
+//        });
+//
+//        var annotated = annotationService.annotate(AnnotateRequest.builder().fileId("fileId").dossierId("dossierId").build());
+//        annotated.getDocument();
+//
+//        File tempFile = File.createTempFile("annotated",".pdf");
+//        IOUtils.write(annotated.getDocument(), new FileOutputStream(tempFile));
+//        log.warn("File saved to: {}",tempFile.getAbsolutePath());
+//        Runtime.getRuntime().exec("open "+tempFile.getAbsolutePath());
     }
 
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.realdata;
 
-
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
@@ -37,6 +36,7 @@ import org.springframework.test.context.junit4.SpringRunner;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
@@ -114,9 +114,8 @@ public class LiveDataIntegrationTest {
         ObjectMapper objectMapper = new ObjectMapper();
 
         var jsonNode = objectMapper.readTree(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "types.json").getInputStream());
-        types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(
-                new TypeReference<List<Type>>() {
-                }));
+        types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(new TypeReference<List<Type>>() {
+        }));
 
         types.forEach(t -> {
             t.setId(t.getType());
@@ -126,27 +125,29 @@ public class LiveDataIntegrationTest {
         when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(types);
         when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>());
 
-
         when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(), Colors.class));
 
         when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
             String typeName = answer.getArgument(0);
 
             var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
-            if(found.isPresent()) {
+            if (found.isPresent()) {
                 var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
                 type.setEntries(getEntries(typeName, type.getTypeId()));
+                type.setFalsePositiveEntries(getFalsePositiveEntries(typeName, type.getTypeId()));
 
                 return type;
-            }else{
+            } else {
                 return null;
             }
         });
 
-        dictionaryService.updateDictionary("dossierTemplateId","dossierId");
+        dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
     }
 
+
     public void simulateIncrement(List<String> values, String deltaTypeName, long version) {
+
         when(dictionaryClient.getVersion(anyString())).thenReturn(version);
         when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(version);
 
@@ -154,7 +155,7 @@ public class LiveDataIntegrationTest {
             String typeName = answer.getArgument(0);
 
             var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
-            if(found.isPresent()) {
+            if (found.isPresent()) {
 
                 var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
 
@@ -165,7 +166,7 @@ public class LiveDataIntegrationTest {
                 }
 
                 return type;
-            }else{
+            } else {
                 return null;
             }
         });
@@ -174,17 +175,20 @@ public class LiveDataIntegrationTest {
 
     @Test
     public void testUpdateDictionary() {
+
         dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
 
         var dict = dictionaryService.getDeepCopyDictionary("dossierTemplateId", "dossierId");
         assertThat(dict.getLocalAccessMap().size()).isEqualTo(12);
     }
 
+
     @SneakyThrows
     private List<DictionaryEntry> getEntries(String typeName, String typeId) {
+
         Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
         for (var resource : dictionaryResources) {
-            if (resource.getFilename().contains(typeName)) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && !resource.getFilename().contains("false_positive")) {
 
                 List<String> lines = IOUtils.readLines(resource.getInputStream());
                 return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
@@ -193,4 +197,20 @@ public class LiveDataIntegrationTest {
         }
         return new ArrayList<>();
     }
+
+    @SneakyThrows
+    private List<DictionaryEntry> getFalsePositiveEntries(String typeName, String typeId) {
+
+        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
+        for (var resource : dictionaryResources) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName) && resource.getFilename().contains("false_positive")) {
+
+                List<String> lines = IOUtils.readLines(resource.getInputStream());
+                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
+
+            }
+        }
+        return new ArrayList<>();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java

@@ -1,15 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import org.apache.commons.lang3.StringUtils;
-import org.junit.Test;
-
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
 
 public class RegExPatternTest {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -1,23 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;
 
-import com.amazonaws.services.s3.AmazonS3;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
-import org.junit.Ignore;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import org.junit.Test;
-import org.junit.platform.commons.util.StringUtils;
 import org.junit.runner.RunWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
@@ -31,20 +25,20 @@ import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import javax.imageio.ImageIO;
-import java.io.ByteArrayOutputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils.getTemporaryDirectory;
-import static org.assertj.core.api.Assertions.assertThat;
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
 
 import lombok.SneakyThrows;
 
@@ -89,15 +83,18 @@ public class PdfSegmentationServiceTest {
 
     @Test
     @SneakyThrows
-    public void testMapping(){
+    public void testMapping() {
+
         ClassPathResource responseJson = new ClassPathResource("files/image_response.json");
         ImageServiceResponse imageServiceResponse = objectMapper.readValue(responseJson.getInputStream(), ImageServiceResponse.class);
 
         Map<Integer, List<PdfImage>> images = new HashMap<>();
-        imageServiceResponse.getData().stream().forEach(imageMetadata -> {
-            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
-                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)), imageMetadata.getPosition().getPageNumber()));
-        });
+        imageServiceResponse.getData()
+                .forEach(imageMetadata -> images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
+                        .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry()
+                                .getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification()
+                                .getLabel()
+                                .toUpperCase(Locale.ROOT)), imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber())));
 
         System.out.println("object");
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/stringmatching/StringMatchingPerformanceTest.java

@@ -0,0 +1,85 @@
+package com.iqser.red.service.redaction.v1.server.stringmatching;
+
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+@RunWith(SpringRunner.class)
+public class StringMatchingPerformanceTest {
+
+    @Test
+    @SneakyThrows
+    public void testStringPerformance() {
+
+        String text = IOUtils.toString(new ClassPathResource("stringmatching/hamlet.txt").getInputStream()).toLowerCase();
+        Set<String> dictionary = IOUtils.readLines(new ClassPathResource("stringmatching/names.txt").getInputStream())
+                .stream()
+                .map(String::toLowerCase).collect(Collectors.toSet());
+
+        System.out.println("Loaded text has a length of " + text.length() + " symbols");
+        System.out.println("Dictionary has " + dictionary.size() + " entries");
+
+        var patterns = dictionary.stream()
+                .map(p -> Pattern.compile(Pattern.quote(p))).collect(Collectors.toList());
+        var trie = Trie.builder().ignoreCase().addKeywords(dictionary).build();
+
+
+        // 1. Naive approach
+        long t1 = System.currentTimeMillis();
+        var naiveIndexes = new HashSet<Index>();
+        for (var entry : dictionary) {
+            var startIndex = 0;
+            do {
+                startIndex = text.indexOf(entry, startIndex + 1);
+                if (startIndex != -1) {
+                    naiveIndexes.add(new Index(startIndex, startIndex + entry.length()));
+                }
+            } while (startIndex != -1);
+        }
+        long t2 = System.currentTimeMillis();
+        System.out.println("Naive approach found " + naiveIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // 2. Boyer Moore
+        t1 = System.currentTimeMillis();
+        var boyerMooreIndexes = new HashSet<Index>();
+        for (var pattern : patterns) {
+            boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).collect(Collectors.toList()));
+        }
+        t2 = System.currentTimeMillis();
+        System.out.println("Boyer Moore found " + boyerMooreIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // 3. Aho Corasick
+        t1 = System.currentTimeMillis();
+        var result = trie.parseText(text);
+        var ahoCorasickIndexes = result.stream().map(r -> new Index(r.getStart(), r.getEnd() + 1)).collect(Collectors.toSet());
+        t2 = System.currentTimeMillis();
+        System.out.println("Aho Corasick found " + ahoCorasickIndexes.size() + " entries in " + (t2 - t1) + "ms");
+
+
+        // Assert that all algorithms are equal
+        assertThat(naiveIndexes).isEqualTo(boyerMooreIndexes).isEqualTo(ahoCorasickIndexes);
+
+    }
+
+    @AllArgsConstructor
+    @EqualsAndHashCode(of = {"start", "end"})
+    public static class Index {
+        int start;
+        int end;
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/files/Cyprodinil/39 Cyprodinil - EU AIR3 - Document F - Notification submitted to the Commission.json

@@ -0,0 +1 @@
+{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"12ac3d26ce82a77e469c64d56081662d","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Table in: COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.8,"y":651.16003},"width":54.039078,"height":-11.520004,"page":4}],"sectionNumber":3,"textBefore":"Report: Document F/02, ","textAfter":". (2014). Cyprodinil","comments":[],"startOffset":23,"endOffset":35,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:36:30.044122+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dd614353319e7c5a8cf8b3346dc12fac","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":306.84,"y":679.12},"width":59.71997,"height":-12.0,"page":4}],"sectionNumber":5,"textBefore":"F report (","textAfter":"., 2014).","comments":[],"startOffset":247,"endOffset":259,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":"2022-08-24T16:36:30.044134+02:00"}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"hint":false,"recommendation":false,"falsePositive":false,"image":false,"dictionaryEntry":true,"dossierDictionaryEntry":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}