Pull request #432 : RED-4686 - storage commons update

Merge in RED/redaction-service from RED-4686 to master * commit '24612e347171f27eb8747772b82d13c98f40344b': RED-4686 - storage commons update - added annotations and test-file RED-4686 - storage commons update
RED-4686 - storage commons update - added annotations and test-file
2022-07-25 21:19:43 +02:00 · 2022-07-25 22:08:36 +03:00 · 2022-07-25 21:45:58 +03:00 · 2022-07-25 16:11:37 +02:00 · 2022-07-25 16:08:23 +02:00 · 2022-07-25 16:04:51 +02:00
39 changed files with 2300 additions and 1210 deletions
@@ -26,3 +26,4 @@
 **/.DS_Store
 **/classpath-data.json
 **/dependencies-and-licenses-overview.txt
+/redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/
@@ -1,10 +1,12 @@
 package buildjob;

-import java.time.DayOfWeek;
+import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+
 import java.time.LocalTime;

 import com.atlassian.bamboo.specs.api.BambooSpec;
 import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.Variable;
 import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
 import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
 import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
@@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.api.builders.Variable;
+import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
 import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
 import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
-import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
 import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
 import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
-import com.atlassian.bamboo.specs.util.BambooServer;
-import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
-
-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+import com.atlassian.bamboo.specs.util.BambooServer;

 /**
 * Plan configuration for Bamboo.
@@ -41,14 +39,15 @@ public class PlanSpec {

    private static final String SERVICE_NAME = "redaction-service";

-    private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
+    private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";

    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");

+
    /**
     * Run main to publish plan on Bamboo
     */
-    public static void main(final String[] args) throws Exception {
+    public static void main(final String[] args) {
        //By default credentials are read from the '.credentials' file.
        BambooServer bambooServer = new BambooServer("http://localhost:8085");

@@ -57,15 +56,21 @@ public class PlanSpec {
        PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
        bambooServer.publish(planPermission);

+        Plan nightPlan = new PlanSpec().createNightPlan();
+        bambooServer.publish(nightPlan);
+        PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
+        bambooServer.publish(nightPlanPermission);
+
        Plan secPlan = new PlanSpec().createSecBuild();
        bambooServer.publish(secPlan);
        PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
        bambooServer.publish(secPlanPermission);
    }

+
    private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
-        Permissions permission = new Permissions()
-                .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
+
+        Permissions permission = new Permissions().userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
                .groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
                .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
                .loggedInUserPermissions(PermissionType.VIEW)
@@ -73,105 +78,80 @@ public class PlanSpec {
        return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
    }

+
    private Project project() {
-        return new Project()
-                .name("RED")
-                .key(new BambooKey("RED"));
+
+        return new Project().name("RED").key(new BambooKey("RED"));
    }

+
    public Plan createPlan() {
-        return new Plan(
-                project(),
-                SERVICE_NAME, new BambooKey(SERVICE_KEY))
-                .description("Plan created from (enter repository url of your plan)")
+
+        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
                .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage")
-                        .jobs(new Job("Default Job",
-                                new BambooKey("JOB1"))
-                                .tasks(
-                                        new ScriptTask()
-                                                .description("Clean")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-                                                        "rm -rf ./*"),
-                                        new VcsCheckoutTask()
-                                                .description("Checkout Default Repository")
-                                                .cleanCheckout(true)
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
-                                        new ScriptTask()
-                                                .description("Build")
-                                                .location(Location.FILE)
-                                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                                .argument(SERVICE_NAME),
-                                        createJUnitParserTask()
-                                                .description("Resultparser")
-                                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                                .enabled(true),
-                                        new InjectVariablesTask()
-                                                .description("Inject git Tag")
-                                                .path("git.tag")
-                                                .namespace("g")
-                                                .scope(InjectVariablesScope.LOCAL),
-                                        new VcsTagTask()
-                                                .description("${bamboo.g.gitTag}")
-                                                .tagName("${bamboo.g.gitTag}")
-                                                .defaultRepository())
-                                .dockerConfiguration(
-                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
-                                )
-                        )
-                )
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                .enabled(true), new InjectVariablesTask().description("Inject git Tag")
+                                .path("git.tag")
+                                .namespace("g")
+                                .scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                .linkedRepositories("RED / " + SERVICE_NAME)

                .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement()
-                        .createForVcsBranch()
-                        .delete(new BranchCleanup()
-                                .whenInactiveInRepositoryAfterDays(14))
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
+                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
                        .notificationForCommitters());
    }

-    public Plan createSecBuild() {
-        return new Plan(
-                project(),
-                SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
-                .description("Security Analysis Plan")
-                .stages(new Stage("Default Stage")
-                        .jobs(new Job("Default Job",
-                                new BambooKey("JOB1"))
-                                .tasks(
-                                        new ScriptTask()
-                                                .description("Clean")
-                                                .inlineBody("#!/bin/bash\n" +
-                                                        "set -e\n" +
-                                                        "rm -rf ./*"),
-                                        new VcsCheckoutTask()
-                                                .description("Checkout Default Repository")
-                                                .cleanCheckout(true)
-                                                .checkoutItems(new CheckoutItem().defaultRepository()),
-                                        new ScriptTask()
-                                                .description("Sonar")
-                                                .location(Location.FILE)
-                                                .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
-                                                .argument(SERVICE_NAME))
-                                .dockerConfiguration(
-                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .dockerRunArguments("--net=host")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
-                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
-                                )
-                        )
-                )
+
+    public Plan createNightPlan() {
+
+        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
+                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
+                                .enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                .argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
+                                .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                .enabled(true))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(
-                        new ScheduledTrigger()
-                                .scheduleOnceDaily(LocalTime.of(23, 00)))
-                .planBranchManagement(new PlanBranchManagement()
-                        .createForVcsBranchMatching("release.*")
-                        .notificationForCommitters());
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
    }
+
+
+    public Plan createSecBuild() {
+
+        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
+                                .cleanCheckout(true)
+                                .checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Sonar")
+                                .location(Location.FILE)
+                                .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
+                                .argument(SERVICE_NAME))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .dockerRunArguments("--net=host")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
+    }
+
 }
@@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
 public class PlanSpecTest {
    @Test
    public void checkYourPlanOffline() throws PropertiesValidationException {
+
        Plan plan = new PlanSpec().createPlan();
        EntityPropertiesBuilders.build(plan);

+        Plan nightPlan = new PlanSpec().createNightPlan();
+        EntityPropertiesBuilders.build(nightPlan);
+
        Plan secPlan = new PlanSpec().createSecBuild();
        EntityPropertiesBuilders.build(secPlan);
    }
@@ -29,15 +29,10 @@

    <dependencyManagement>
        <dependencies>
-            <dependency>
-                <groupId>com.dslplatform</groupId>
-                <artifactId>dsl-json-java8</artifactId>
-                <version>${dsljson.version}</version>
-            </dependency>
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.13.0</version>
+                <version>1.17.0</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -12,7 +12,7 @@
    <artifactId>redaction-service-api-v1</artifactId>

    <properties>
-        <persistence-service.version>1.166.0</persistence-service.version>
+        <persistence-service.version>1.240.0</persistence-service.version>
    </properties>

    <dependencies>
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;

+import com.dslplatform.json.CompiledJson;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -11,6 +12,7 @@ import java.util.Map;

@Data
@Builder
+@CompiledJson
@NoArgsConstructor
@AllArgsConstructor
 public class ImportedRedactions {
@@ -38,6 +38,12 @@
            <version>${jackson.version}</version>
        </dependency>

+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
+
        <dependency>
            <groupId>org.ahocorasick</groupId>
            <artifactId>ahocorasick</artifactId>
@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import com.dslplatform.json.CompiledJson;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@CompiledJson
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedSectionText {
+
+    private int sectionNumber;
+    private String text;
+
+}
@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.dslplatform.json.CompiledJson;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@CompiledJson
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedText {
+
+    private int numberOfPages;
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+
+}
@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.client;

+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
 import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 import org.springframework.util.FileCopyUtils;
 import org.springframework.web.multipart.MultipartFile;

-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-
 public class MockMultipartFile implements MultipartFile {

    private final String name;
@@ -82,13 +82,13 @@ public class MockMultipartFile implements MultipartFile {
    }


-    public byte[] getBytes() throws IOException {
+    public byte[] getBytes() {

        return this.content;
    }


-    public InputStream getInputStream() throws IOException {
+    public InputStream getInputStream() {

        return new ByteArrayInputStream(this.content);
    }
@@ -5,8 +5,8 @@ import java.util.List;
 import java.util.Map;

 import com.dslplatform.json.CompiledJson;
+
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;

@@ -16,6 +16,6 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
 public class NerEntities {

-   private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
+    private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();

 }
@@ -301,12 +301,24 @@ class LegacyPDFStreamEngine extends PDFStreamEngine
            nextY -= pageSize.getLowerLeftY();
        }

-        processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
-                pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
-                Math.abs(dyDisplay), dxDisplay,
-                Math.abs(spaceWidthDisplay), unicodeMapping, new int[] { code }, font,
-                fontSize,
-                (int)(fontSize * textMatrix.getScalingFactorX())));
+        // This is a hack for unicode letter with 2 chars e.g. RA see unicodeProblem.pdf
+        if(unicodeMapping.length() == 2){
+            processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
+                    pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
+                    Math.abs(dyDisplay), dxDisplay,
+                    Math.abs(spaceWidthDisplay), Character.toString(unicodeMapping.charAt(0)), new int[] { code }, font,
+                    fontSize,
+                    (int)(fontSize * textMatrix.getScalingFactorX())));
+            processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
+                    pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
+                    Math.abs(dyDisplay), dxDisplay,
+                    Math.abs(spaceWidthDisplay), Character.toString(unicodeMapping.charAt(1)), new int[] { code }, font,
+                    fontSize,
+                    (int)(fontSize * textMatrix.getScalingFactorX())));
+        } else {
+
+            processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(), pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY, Math.abs(dyDisplay), dxDisplay, Math.abs(spaceWidthDisplay), unicodeMapping, new int[]{code}, font, fontSize, (int) (fontSize * textMatrix.getScalingFactorX())));
+        }
    }

    /**
@@ -5,6 +5,7 @@ import org.springframework.beans.BeanUtils;

 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
+import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonIgnore;

 import lombok.AllArgsConstructor;
@@ -26,7 +27,13 @@ public class RedTextPosition {
    private float pageHeight;
    private float pageWidth;
    private String unicode;
+
+    @JsonAlias("xdirAdj")
+    @JsonAttribute(alternativeNames = {"xdirAdj"})
    private float XDirAdj;
+
+    @JsonAlias("ydirAdj")
+    @JsonAttribute(alternativeNames = {"ydirAdj"})
    private float YDirAdj;
    private float width;
    private float heightDir;
@@ -63,4 +70,14 @@ public class RedTextPosition {
    }


+    @JsonAlias("xdirAdj")
+    @JsonAttribute(alternativeNames = {"xdirAdj"})
+    public void setXDirAdj(float XDirAdj) {this.XDirAdj = XDirAdj;}
+
+
+    @JsonAlias("ydirAdj")
+    @JsonAttribute(alternativeNames = {"ydirAdj"})
+    public void setYDirAdj(float YDirAdj) {this.YDirAdj = YDirAdj;}
+
+
 }
@@ -137,8 +137,7 @@ public class TextPositionSequence implements CharSequence {
        if (textPositions.get(0).getRotation() == 90) {
            return textPositions.get(0).getYDirAdj();
        } else {
-            return textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
+            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
        }
    }

@@ -262,8 +261,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public Rectangle getRectangle() {

-        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0)
-                .getRotation(), textPositions.get(0).getDir());
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, textPositions.get(0).getRotation(), textPositions.get(0).getDir());

        float height = getTextHeight();

@@ -275,36 +273,30 @@ public class TextPositionSequence implements CharSequence {
        if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 90f) {

            posYInit = getX1();
-            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
-                    .getWidthDirAdj() - 3;
+            posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
            posXInit = textPositions.get(0).getYDirAdj() + 2;
            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;

        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 180f) {

            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;

        } else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) {

            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;

        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 0.0f) {

-            posXInit = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
+            posXInit = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
            posXEnd = textPositions.get(0).getXDirAdj();
            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;

        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 90.0f) {
            posXEnd = textPositions.get(0).getYDirAdj() + 2;
@@ -313,29 +305,23 @@ public class TextPositionSequence implements CharSequence {

        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 180.0f) {

-            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() - 4;
+            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getXDirAdj() - 4;
            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getXDirAdj();
-            posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1)
-                    .getHeightDir();
-            posYEnd = textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();
+            posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1).getHeightDir();
+            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();

        } else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 270.0f) {

            posXInit = textPositions.get(0).getPageWidth() - getX1();
            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
            posYInit = textPositions.get(0).getPageHeight() - getY1();
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() - height - 4;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getXDirAdj() - height - 4;

        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 0f) {

-            posXEnd = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
+            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;

        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 90f) {

@@ -347,27 +333,23 @@ public class TextPositionSequence implements CharSequence {
        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {

            posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
-            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
            posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;

        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 270.0f) {

            posYInit = textPositions.get(0).getPageHeight() - getX1();
-            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
-                    .getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
+            posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;

        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) {

            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
            posYEnd = posYInit + 1;
            posXInit = textPositions.get(0).getXDirAdj();
-            posXEnd = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
+            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;

        } else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 90.0f) {

@@ -388,16 +370,13 @@ public class TextPositionSequence implements CharSequence {
            posYInit = textPositions.get(0).getPageHeight() - getX1();
            posYEnd = textPositions.get(0).getPageHeight() - getX2() - height;
            posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
-            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + height;
+            posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;

        } else {
            // page rotation = 0 and text direction = 0
-            posXEnd = textPositions.get(textPositions.size() - 1)
-                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
+            posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
            posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
-            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
-                    .getYDirAdj() + 2;
+            posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
        }

        var rectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
@@ -1,5 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;

+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@@ -8,13 +14,9 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.stereotype.Service;
-
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;

@Slf4j
@Service
@@ -55,6 +57,9 @@ public class RedactionMessageReceiver {
                    result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
                    log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
                    break;
+
+                default:
+                    throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
            }

            result.setMessageType(analyzeRequest.getMessageType());
@@ -643,7 +643,7 @@ public class Section {
        while (matcher.find()) {
            String match = matcher.group(group);
            if (StringUtils.isNotBlank(match) && match.length() >= 3) {
-                localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
+                localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(match);
            }
        }
    }
@@ -801,7 +801,7 @@ public class Section {
                }

                if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) {
-                    localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(cleanValue);
+                    localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(cleanValue);
                }
            }
        }
@@ -865,7 +865,7 @@ public class Section {
            if (StringUtils.isNotBlank(match) && match.length() >= 3) {
                Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
                EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
-                localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
+                localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(match);
            }
        }
    }
@@ -980,7 +980,7 @@ public class Section {
                    EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);

                    if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
                    }
                }
            }
@@ -1018,7 +1018,7 @@ public class Section {
                    EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);

                    if (redactEverywhere && !isLocal()) {
-                        localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
+                        localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
                    }
                }
            }
@@ -1046,7 +1046,7 @@ public class Section {
                        EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);

                        if (redactEverywhere && !isLocal()) {
-                            localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
+                            localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(line.trim());
                        }
                    }
                }
@@ -21,4 +21,9 @@ public class ImageServiceResponse {
    @JsonAttribute(alternativeNames = {"imageMetadata"})
    private List<ImageMetadata> data = new ArrayList<>();

+    @JsonProperty(value = "imageMetadata")
+    @JsonAlias("data")
+    @JsonAttribute(alternativeNames = {"imageMetadata"})
+    public void setData(List<ImageMetadata> data) {this.data = data;}
+
 }
@@ -1,5 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+import org.springframework.web.bind.annotation.RequestBody;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
@@ -8,15 +24,31 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.entity
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedSectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
@@ -28,15 +60,6 @@ import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-import org.springframework.web.bind.annotation.RequestBody;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;

@Slf4j
@Service
@@ -93,8 +116,9 @@ public class AnalyzeService {
                        .map(SectionArea::getPage)
                        .collect(Collectors.toSet()), sectionText.getSectionAreas())));

-        log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        log.info("Store text, simplified text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());

        return AnalyzeResult.builder()
@@ -312,4 +336,17 @@ public class AnalyzeService {
        }
    }

+
+    private SimplifiedText convert(Text text) {
+
+        List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+        int numberOfPages = 0;
+
+        if (text != null) {
+            numberOfPages = text.getNumberOfPages();
+            text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText())));
+        }
+        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
+    }
+
 }
@@ -5,12 +5,10 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-
 import feign.FeignException;
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;
@@ -25,6 +23,7 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
 public class DictionaryService {

+    private static final String DEFAULT_COLOR = "#cccccc";
    private final DictionaryClient dictionaryClient;

    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
@@ -178,10 +177,10 @@ public class DictionaryService {

                Colors colors = dictionaryClient.getColors(dossierTemplateId);

-                dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
-                dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
-                dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
-                dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
+                dictionaryRepresentation.setDefaultColor(convertColor(DEFAULT_COLOR));
+                dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAddColor()));
+                dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemoveColor()));
+                dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getSkippedColor()));
                dictionaryRepresentation.setDossierTemplateId(dossierTemplateId);
                dictionaryRepresentation.setDictionaryVersion(version);
                dictionaryRepresentation.setDictionary(dictionary);
@@ -221,20 +221,18 @@ public class EntityRedactionService {

    private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {

-        analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> {
-            analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
+        analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {

-                if (dictionary.getLocalAccessMap().get(key) == null) {
-                    log.warn("Dictionary {} is null", key);
-                }
+            if (dictionary.getLocalAccessMap().get(key) == null) {
+                log.warn("Dictionary {} is null", key);
+            }

-                if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
-                    log.warn("Dictionary {} localEntries is null", key);
-                }
+            if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
+                log.warn("Dictionary {} localEntries is null", key);
+            }

-                dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
-            });
-        });
+            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
+        }));
    }


@@ -115,7 +115,7 @@ public class ImportedRedactionService {
        Point p = new Point();

        if (rectangle.getWidth() < 0) {
-            p.setX(rectangle.getTopLeft().getX() - rectangle.getWidth());
+            p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
            r.setWidth(Math.abs(rectangle.getWidth()));
        } else {
            p.setX(rectangle.getTopLeft().getX());
@@ -127,6 +127,7 @@ public class ImportedRedactionService {
            r.setHeight(Math.abs(rectangle.getHeight()));
        } else {
            p.setY(rectangle.getTopLeft().getY());
+            r.setHeight(rectangle.getHeight());
        }

        r.setTopLeft(p);
@@ -1,15 +1,40 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import java.awt.Color;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.ManualChange;
+import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;

@@ -18,20 +43,13 @@ import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.awt.*;
-import java.time.OffsetDateTime;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;

@Slf4j
@Service
@RequiredArgsConstructor
 public class RedactionLogMergeService {

-
+    private static final String DELETED_TYPE_COLOR = "#9398a0";
    private final SectionTextService sectionTextService;

    private final RedactionStorageService redactionStorageService;
@@ -39,6 +57,7 @@ public class RedactionLogMergeService {

    @Timed("redactmanager_getMergedRedactionLog")
    public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
+
        log.debug("Requested preview for: {}", redactionRequest);

        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
@@ -72,15 +91,15 @@ public class RedactionLogMergeService {
        return merged;
    }

-    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
-                                               Set<Integer> excludedPages, List<Type> types, Colors colors) {
+
+    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
+                                               ManualRedactions manualRedactions, Set<Integer> excludedPages,
+                                               List<Type> types, Colors colors) {

        var skippedImportedRedactions = new HashSet<>();
-
        log.info("Merging Redaction log with manual redactions");
        if (manualRedactions != null) {

-
            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);

            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
@@ -112,9 +131,12 @@ public class RedactionLogMergeService {

        Set<String> processedIds = new HashSet<>();
        redactionLog.getRedactionLogEntry().removeIf(entry -> {
+
            if (entry.getImportedRedactionIntersections() != null) {
                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) {
+                if (!entry.getImportedRedactionIntersections()
+                        .isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType()
+                        .equals("image") || entry.getType().equals("ocr")))) {
                    return true;
                }
            }
@@ -179,7 +201,6 @@ public class RedactionLogMergeService {
                String manualOverrideReason = null;
                if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {

-
                    redactionLogEntry.setType(imageRecategorization.getType());
                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());

@@ -196,16 +217,20 @@ public class RedactionLogMergeService {
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
                }

-                redactionLogEntry.setReason(manualOverrideReason);
+                if(manualOverrideReason != null) {
+                    redactionLogEntry.setReason(manualOverrideReason);
+                }

-                redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
-                        .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                        .withChange("type", imageRecategorization.getType()));
+                redactionLogEntry.getManualChanges()
+                        .add(ManualChange.from(imageRecategorization)
+                                .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
+                                .withChange("type", imageRecategorization.getType()));
            }

            if (mrw.getItem() instanceof IdRemoval) {
                var manualRemoval = (IdRemoval) mrw.getItem();
-                if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
+                if (manualRemoval.getStatus()
+                        .equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
                    log.debug("Skipping merge for dictionary-modifying entry");
                } else {
                    String manualOverrideReason = null;
@@ -219,11 +244,13 @@ public class RedactionLogMergeService {
                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                    }

-                    redactionLogEntry.setReason(manualOverrideReason);
+                    if(manualOverrideReason != null) {
+                        redactionLogEntry.setReason(manualOverrideReason);
+                    }

-                    redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
-                            .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
-                                    ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
+                    redactionLogEntry.getManualChanges()
+                            .add(ManualChange.from(manualRemoval)
+                                    .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
                }
            }

@@ -247,7 +274,10 @@ public class RedactionLogMergeService {
                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                }

-                redactionLogEntry.setReason(manualOverrideReason);
+                if(manualOverrideReason != null) {
+                    redactionLogEntry.setReason(manualOverrideReason);
+                }
+
                var manualChange = ManualChange.from(manualForceRedact)
                        .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);

@@ -273,7 +303,9 @@ public class RedactionLogMergeService {
                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                }

-                redactionLogEntry.setReason(manualOverrideReason);
+                if(manualOverrideReason != null) {
+                    redactionLogEntry.setReason(manualOverrideReason);
+                }

                var manualChange = ManualChange.from(manualLegalBasisChange)
                        .withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
@@ -296,7 +328,7 @@ public class RedactionLogMergeService {
                    redactionLogEntry.setValue(manualResizeRedact.getValue());

                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
+                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
                    }
@@ -307,16 +339,17 @@ public class RedactionLogMergeService {
                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));

                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if(manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
+                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
                    }
                }

                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
-                        .withManualRedactionType(ManualRedactionType.RESIZE)
-                        .withChange("value", manualResizeRedact.getValue()));
+                redactionLogEntry.getManualChanges()
+                        .add(ManualChange.from(manualResizeRedact)
+                                .withManualRedactionType(ManualRedactionType.RESIZE)
+                                .withChange("value", manualResizeRedact.getValue()));
            }

        });
@@ -338,7 +371,8 @@ public class RedactionLogMergeService {


    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
-                                                       Map<String, List<Comment>> comments, Colors colors, List<Type> types) {
+                                                       Map<String, List<Comment>> comments, Colors colors,
+                                                       List<Type> types) {

        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();

@@ -360,9 +394,12 @@ public class RedactionLogMergeService {
        return redactionLogEntries;
    }

+
    private List<RedactionLogComment> convert(List<Comment> comments) {
-        return comments == null ? null :comments.stream().map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(),
-                c.getFileId(), c.getDate(), c.getSoftDeletedTime())).collect(Collectors.toList());
+
+        return comments == null ? null : comments.stream()
+                .map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
+                .collect(Collectors.toList());
    }


@@ -377,8 +414,8 @@ public class RedactionLogMergeService {

    @SuppressWarnings("PMD.UselessParentheses")
    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
-                || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null);
+
+        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
    }


@@ -412,14 +449,14 @@ public class RedactionLogMergeService {
    }


-    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction,
-                             boolean skipped, List<Type> types) {
+    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped,
+                             List<Type> types) {

        if (requested) {
-            return convertColor(colors.getRequestRemove());
+            return convertColor(colors.getRequestRemoveColor());
        }
        if (skipped || !isRedaction && !isHint(types, type)) {
-            return convertColor(colors.getNotRedacted());
+            return convertColor(colors.getSkippedColor());
        }
        return getColor(types, type);
    }
@@ -428,9 +465,9 @@ public class RedactionLogMergeService {
    private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {

        if (status.equals(AnnotationStatus.REQUESTED)) {
-            return convertColor(colors.getRequestAdd());
+            return convertColor(colors.getRequestAddColor());
        } else if (status.equals(AnnotationStatus.DECLINED)) {
-            return convertColor(colors.getNotRedacted());
+            return convertColor(colors.getSkippedColor());
        }
        return getColor(types, type);
    }
@@ -444,7 +481,8 @@ public class RedactionLogMergeService {
            return convertColor(foundAndNotDeletedType.get().getHexColor());
        }
        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0"));
+        return firstDeletedType.map(value -> convertColor(value.getHexColor()))
+                .orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
    }


@@ -466,11 +504,15 @@ public class RedactionLogMergeService {
        return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
    }

+
    private List<Type> getMatchingTypes(List<Type> types, String type) {
+
        return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
    }

+
    private boolean isDeletedType(Type type) {
+
        return type.getSoftDeletedTime() != null;
    }

@@ -1,18 +1,28 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;

-import io.micrometer.core.annotation.Timed;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

-import java.util.*;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
@@ -93,33 +103,31 @@ public class EntitySearchUtils {
    }


-    private Set<Entity> applyResizeRedactions(Set<Entity> entitiesWithPositions, ManualRedactions manualRedactions) {
+    private void applyResizeRedactions(Set<Entity> entitiesWithPositions, ManualRedactions manualRedactions) {

-        if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()){
-            return entitiesWithPositions;
+        if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()) {
+            return;
        }

-        entitiesWithPositions.forEach(e -> e.getPositionSequences().forEach(pos -> {
-            manualRedactions.getResizeRedactions().stream().filter(resize -> resize.getStatus().equals(AnnotationStatus.APPROVED)).forEach(resize -> {
-                if (resize.getAnnotationId().equals(pos.getId())) {
-                    if (resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
-                        int start = e.getWord().indexOf(resize.getValue());
-                        e.setStart(e.getStart() + start);
-                        e.setEnd(e.getStart() + resize.getValue().length());
-                        e.setResized(true);
-                        e.setWord(resize.getValue());
-                    } else if(resize.getValue().length() > e.getWord().length() && resize.getValue().contains(e.getWord())){
-                        int start = resize.getValue().indexOf(e.getWord());
-                        e.setStart(e.getStart() - start);
-                        e.setEnd(e.getStart() + resize.getValue().length());
-                        e.setResized(true);
-                        e.setWord(resize.getValue());
+        entitiesWithPositions.forEach(e -> e.getPositionSequences()
+                .forEach(pos -> manualRedactions.getResizeRedactions().stream().filter(resize -> resize.getStatus().equals(AnnotationStatus.APPROVED)).forEach(resize -> {
+                    if (resize.getAnnotationId().equals(pos.getId())) {
+                        if (resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
+                            int start = e.getWord().indexOf(resize.getValue());
+                            e.setStart(e.getStart() + start);
+                            e.setEnd(e.getStart() + resize.getValue().length());
+                            e.setResized(true);
+                            e.setWord(resize.getValue());
+                        } else if (resize.getValue().length() > e.getWord().length() && resize.getValue().contains(e.getWord())) {
+                            int start = resize.getValue().indexOf(e.getWord());
+                            e.setStart(e.getStart() - start);
+                            e.setEnd(e.getStart() + resize.getValue().length());
+                            e.setResized(true);
+                            e.setWord(resize.getValue());
+                        }
                    }
-                }
-            });
-        }));
+                })));

-        return entitiesWithPositions;
    }


@@ -1,27 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
-import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
-import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.SystemUtils;
-import org.apache.pdfbox.io.MemoryUsageSetting;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.springframework.stereotype.Service;
-
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -35,6 +13,30 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.pdfbox.io.MemoryUsageSetting;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
+import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
@Service
@RequiredArgsConstructor
@@ -72,7 +74,7 @@ public class PdfSegmentationService {
                Document document = new Document();
                List<Page> pages = new ArrayList<>();

-                pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(1024 * 1024 * 64));
+                pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(67108864L));
                pdDocument.setAllSecurityToBeRemoved(true);
                long pageCount = pdDocument.getNumberOfPages();

@@ -12,17 +12,14 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import io.micrometer.core.annotation.Timed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;

-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;

@@ -32,10 +29,6 @@ import java.io.InputStream;
 public class RedactionStorageService {

    private final StorageService storageService;
-    private final ObjectMapper objectMapper;
-
-    private final DslJson<Object> dslJson = new DslJson<>(Settings.basicSetup());
-

    @SneakyThrows
    public InputStream getStoredObject(String storageId) {
@@ -54,154 +47,66 @@ public class RedactionStorageService {
    @SneakyThrows
    @Timed("redactmanager_storeObject")
    public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
-
-        var bytes = serializeObject(any);
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), bytes);
+        storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
    }

-
-    @SneakyThrows
-    @Timed("redactmanager_serializeObject")
-    private byte[] serializeObject(Object any) {
-
-        try {
-            var baos = new ByteArrayOutputStream();
-            dslJson.serialize(any, baos);
-            return baos.toByteArray();
-        } catch (com.dslplatform.json.SerializationException e) {
-            // Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
-            var bytes = objectMapper.writeValueAsBytes(any);
-            dslJson.newWriter();
-            return bytes;
-        }
-    }
-
-
    @Timed("redactmanager_getImportedRedactions")
    public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {

-        InputStreamResource inputStreamResource;
        try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
        } catch (StorageObjectDoesNotExist e) {
            log.debug("Imported redactions not available.");
            return null;
        }
-
-        return deserializeImportedRedactions(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeImportedRedactions")
-    private ImportedRedactions deserializeImportedRedactions(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not imported redactions", e);
-        }
    }


    @Timed("redactmanager_getRedactionLog")
    public RedactionLog getRedactionLog(String dossierId, String fileId) {

-        InputStreamResource inputStreamResource;
        try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),RedactionLog.class);
        } catch (StorageObjectDoesNotExist e) {
            log.debug("Text not available.");
            return null;
        }

-        return deserializeRedactionLog(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeRedactionLog")
-    private RedactionLog deserializeRedactionLog(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert RedactionLog", e);
-        }
    }


    @Timed("redactmanager_getText")
    public Text getText(String dossierId, String fileId) {

-        InputStreamResource inputStreamResource;
        try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT),Text.class);
        } catch (StorageObjectDoesNotExist e) {
            log.debug("Text not available.");
            return null;
        }
-
-        return deserializeText(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeText")
-    private Text deserializeText(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(Text.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert Text", e);
-        }
    }


    @Timed("redactmanager_getNerEntities")
    public NerEntities getNerEntities(String dossierId, String fileId) {

-        InputStreamResource inputStreamResource;
        try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
        } catch (StorageObjectDoesNotExist e) {
            throw new NotFoundException("NER Entities are not available.");
        }
-
-        return deserializeNerEntities(inputStreamResource);
    }


-    @Timed("redactmanager_deserializeNerEntities")
-    private NerEntities deserializeNerEntities(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert NER Entities", e);
-        }
-    }
-

    @Timed("redactmanager_getSectionGrid")
    public SectionGrid getSectionGrid(String dossierId, String fileId) {

-        InputStreamResource inputStreamResource;
        try {
-            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
+            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID),SectionGrid.class);
        } catch (StorageObjectDoesNotExist e) {
            throw new NotFoundException("Section Grid is not available.");
        }
-
-        return deserializeSectionGrid(inputStreamResource);
-    }
-
-
-    @Timed("redactmanager_deserializeSectionGrid")
-    private SectionGrid deserializeSectionGrid(InputStreamResource inputStreamResource) {
-
-        try {
-            return dslJson.deserialize(SectionGrid.class, inputStreamResource.getInputStream());
-        } catch (IOException e) {
-            throw new RuntimeException("Could not convert SectionGrid", e);
-        }
    }


@@ -62,7 +62,7 @@ public class DictionaryServiceTest {

        when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
        when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", "#cccccc",
-                "#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc",
+                "#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc","#cccccc",
                "#cccccc", "#cccccc", "#cccccc", "#cccccc"));

        var type1 = new Type();
@@ -1,7 +1,9 @@
 package com.iqser.red.service.redaction.v1.server;

+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.S3StorageService;
 import com.iqser.red.storage.commons.service.StorageService;
 import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
@@ -11,10 +13,13 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;

-public class FileSystemBackedStorageService implements StorageService{
+public class FileSystemBackedStorageService implements StorageService {

    private final Map<String, File> dataMap = new HashMap<>();

@@ -48,36 +53,54 @@ public class FileSystemBackedStorageService implements StorageService{

    }

-    public List<String> listPaths(){
+    @Override
+    @SneakyThrows
+    public <T> void storeJSONObject(String objectId, T any) {
+        File tempFile = File.createTempFile("test", ".tmp");
+        getMapper().writeValue(new FileOutputStream(tempFile), any);
+        dataMap.put(objectId, tempFile);
+    }
+
+    private ObjectMapper getMapper() {
+        var objectMapper = new ObjectMapper();
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.findAndRegisterModules();
+        return objectMapper;
+    }
+
+    @Override
+    @SneakyThrows
+    public <T> T readJSONObject(String objectId, Class<T> clazz) {
+        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
+            throw new StorageObjectDoesNotExist("Stored object not found");
+        }
+        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
+    }
+
+    public List<String> listPaths() {
        return new ArrayList<>(dataMap.keySet());
    }


-    public List<String> listFilePaths(){
+    public List<String> listFilePaths() {
        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
    }


-    @SneakyThrows
-    @Override
-    public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
-
-        IOUtils.write(data, new FileOutputStream(tempFile));
-
-        dataMap.put(objectId, tempFile);
-    }
-
    @Override
    @SneakyThrows
    public void storeObject(String objectId, InputStream stream) {
        File tempFile = File.createTempFile("test", ".tmp");

-        IOUtils.copy(stream, new FileOutputStream(tempFile));
+        try (var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.copy(stream, fileOutputStream);
+        }

        dataMap.put(objectId, tempFile);
    }

+
    public void clearStorage() {
        this.dataMap.forEach((k, v) -> {
            v.delete();
@@ -30,10 +30,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import lombok.SneakyThrows;
-
-import org.apache.commons.io.IOUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -604,8 +601,7 @@ public class RedactionIntegrationTest {
        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";

        ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
-        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());

        long start = System.currentTimeMillis();
        ClassPathResource pdfFileResource = new ClassPathResource(fileName);
@@ -807,6 +803,30 @@ public class RedactionIntegrationTest {
    }


+    @Test
+    public void testUnicodeProblem() throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        AnalyzeRequest request = prepareStorage("files/new/unicodeProblem.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
    @Test
    public void testRotations() throws IOException {

@@ -873,9 +893,9 @@ public class RedactionIntegrationTest {
        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);

        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                        .manualRedactions(manualRedactions)
-                        .colors(colors)
-                        .types(types)
+                .manualRedactions(manualRedactions)
+                .colors(colors)
+                .types(types)
                .dossierId(TEST_DOSSIER_ID)
                .fileId(TEST_FILE_ID)
                .build());
@@ -1593,10 +1613,9 @@ public class RedactionIntegrationTest {
        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
        rankTypeMap.put(IMPORTED_REDACTION, 200);

-        colors.setDefaultColor("#acfc00");
-        colors.setNotRedacted("#cccccc");
-        colors.setRequestAdd("#04b093");
-        colors.setRequestRemove("#04b093");
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
    }


@@ -1604,8 +1623,7 @@ public class RedactionIntegrationTest {
    private void loadNerForTest() {

        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        var bytes = IOUtils.toByteArray(responseJson.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
    }


@@ -1672,7 +1690,7 @@ public class RedactionIntegrationTest {
        ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");

        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream()));
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());

        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
        AnalyzeResult result = analyzeService.analyze(request);
@@ -1688,11 +1706,11 @@ public class RedactionIntegrationTest {
            if (entry.getValue() == null) {
                return;
            }
-            if (entry.getValue().equals("David")){
-                assertThat(entry.getImportedRedactionIntersections().size()).isEqualTo(1);
+            if (entry.getValue().equals("David")) {
+                assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
            }
-            if (entry.getValue().equals("annotation")){
-                assertThat(entry.getImportedRedactionIntersections().size()).isEqualTo(0);
+            if (entry.getValue().equals("annotation")) {
+                assertThat(entry.getImportedRedactionIntersections()).isEmpty();
            }
        });

@@ -1705,12 +1723,12 @@ public class RedactionIntegrationTest {
    @Test
    public void testExpandByPrefixRegEx() throws IOException {

-        assertThat(dictionary.get(AUTHOR).contains("Robinson"));
-        assertThat(!dictionary.get(AUTHOR).contains("Mrs. Robinson"));
-        assertThat(dictionary.get(AUTHOR).contains("Bojangles"));
-        assertThat(!dictionary.get(AUTHOR).contains("Mr. Bojangles"));
-        assertThat(dictionary.get(AUTHOR).contains("Tambourine Man"));
-        assertThat(!dictionary.get(AUTHOR).contains("Mr. Tambourine Man"));
+        assertThat(dictionary.get(AUTHOR)).contains("Robinson");
+        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mrs. Robinson");
+        assertThat(dictionary.get(AUTHOR)).contains("Bojangles");
+        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Bojangles");
+        assertThat(dictionary.get(AUTHOR)).contains("Tambourine Man");
+        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Tambourine Man");

        String fileName = "files/mr-mrs.pdf";
        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
@@ -1752,9 +1770,7 @@ public class RedactionIntegrationTest {
                .lastProcessed(OffsetDateTime.now())
                .build();

-        var bytes = IOUtils.toByteArray(stream);
-
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);

        return request;

@@ -0,0 +1,681 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static java.util.Map.entry;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.annotation.IfProfileValue;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.Change;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.model.ManualChange;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RulesTest.RulesTestConfiguration.class)
+public class RulesTest {
+
+    private static final String RULES_PATH = "drools/testRules.drl";
+    private static final String RULES = loadFromClassPath(RULES_PATH);
+    private static final String VERTEBRATE = "vertebrate";
+    private static final String ADDRESS = "CBI_address";
+    private static final String AUTHOR = "CBI_author";
+    private static final String SPONSOR = "CBI_sponsor";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
+    private static final String PUBLISHED_INFORMATION = "published_information";
+    private static final String TEST_METHOD = "test_method";
+    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";
+    private static final String LOGO = "logo";
+    private static final String SIGNATURE = "signature";
+    private static final String FORMULA = "formula";
+    private static final String OCR = "ocr";
+    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
+    private static final String IMPORTED_REDACTION = "imported_redaction";
+    private static final String PII = "PII";
+    private static final String RESOURCES_PATH = "src/test/resources/";
+    private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
+    private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    private final static String TEST_DOSSIER_ID = "123";
+    private String TEST_FILE_ID = "123";
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
+    private final Map<String, List<String>> falsePositive = new HashMap<>();
+    private static final Map<String, String> typeColorMap = Map.ofEntries(entry(VERTEBRATE, "#ff85f7"), entry(ADDRESS, "#ffe187"), entry(AUTHOR, "#ffe187"), entry(SPONSOR, "#85ebff"), entry(NO_REDACTION_INDICATOR, "#be85ff"), entry(REDACTION_INDICATOR, "#caff85"), entry(HINT_ONLY, "#abc0c4"), entry(MUST_REDACT, "#fab4c0"), entry(PUBLISHED_INFORMATION, "#85ebff"), entry(TEST_METHOD, "#91fae8"), entry(PII, "#66ccff"), entry(PURITY, "#ffe187"), entry(IMAGE, "#fcc5fb"), entry(OCR, "#fcc5fb"), entry(LOGO, "#ffe187"), entry(FORMULA, "#ffe187"), entry(SIGNATURE, "#ffe187"), entry(IMPORTED_REDACTION, "#fcfbe6"));
+    private static final Map<String, Boolean> hintTypeMap = Map.ofEntries(entry(VERTEBRATE, true), entry(ADDRESS, false), entry(AUTHOR, false), entry(SPONSOR, false), entry(NO_REDACTION_INDICATOR, true), entry(REDACTION_INDICATOR, true), entry(HINT_ONLY, true), entry(MUST_REDACT, true), entry(PUBLISHED_INFORMATION, true), entry(TEST_METHOD, true), entry(PII, false), entry(PURITY, false), entry(IMAGE, true), entry(OCR, true), entry(FORMULA, false), entry(LOGO, false), entry(SIGNATURE, false), entry(DOSSIER_REDACTIONS, false), entry(IMPORTED_REDACTION, false));
+    private static final Map<String, Boolean> caseInSensitiveMap = Map.ofEntries(entry(VERTEBRATE, true), entry(ADDRESS, false), entry(AUTHOR, false), entry(SPONSOR, false), entry(NO_REDACTION_INDICATOR, true), entry(REDACTION_INDICATOR, true), entry(HINT_ONLY, true), entry(MUST_REDACT, true), entry(PUBLISHED_INFORMATION, true), entry(TEST_METHOD, false), entry(PII, false), entry(PURITY, false), entry(IMAGE, true), entry(OCR, true), entry(SIGNATURE, true), entry(LOGO, true), entry(FORMULA, true), entry(DOSSIER_REDACTIONS, false), entry(IMPORTED_REDACTION, false));
+    private static final Map<String, Boolean> recommendationTypeMap = Map.ofEntries(entry(VERTEBRATE, false), entry(ADDRESS, false), entry(AUTHOR, false), entry(SPONSOR, false), entry(NO_REDACTION_INDICATOR, false), entry(REDACTION_INDICATOR, false), entry(HINT_ONLY, false), entry(MUST_REDACT, false), entry(PUBLISHED_INFORMATION, false), entry(TEST_METHOD, false), entry(PII, false), entry(PURITY, false), entry(IMAGE, false), entry(OCR, false), entry(FORMULA, false), entry(SIGNATURE, false), entry(LOGO, false), entry(DOSSIER_REDACTIONS, false), entry(IMPORTED_REDACTION, false));
+    private static final Map<String, Integer> rankTypeMap = Map.ofEntries(entry(PURITY, 155), entry(PII, 150), entry(ADDRESS, 140), entry(AUTHOR, 130), entry(SPONSOR, 120), entry(VERTEBRATE, 110), entry(MUST_REDACT, 100), entry(REDACTION_INDICATOR, 90), entry(NO_REDACTION_INDICATOR, 80), entry(PUBLISHED_INFORMATION, 70), entry(TEST_METHOD, 60), entry(HINT_ONLY, 50), entry(IMAGE, 30), entry(OCR, 29), entry(LOGO, 28), entry(SIGNATURE, 27), entry(FORMULA, 26), entry(DOSSIER_REDACTIONS, 200), entry(IMPORTED_REDACTION, 200));
+    private final Colors colors = new Colors();
+
+    @Autowired
+    private RedactionController redactionController;
+    @Autowired
+    private AnnotationService annotationService;
+    @Autowired
+    private AnalyzeService analyzeService;
+    @Autowired
+    private ObjectMapper objectMapper;
+    @MockBean
+    private RulesClient rulesClient;
+    @MockBean
+    private DictionaryClient dictionaryClient;
+    @Autowired
+    private RedactionStorageService redactionStorageService;
+    @Autowired
+    private StorageService storageService;
+    @Autowired
+    private ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
+    @MockBean
+    private AmazonS3 amazonS3;
+    @MockBean
+    private RabbitTemplate rabbitTemplate;
+    @MockBean
+    private LegalBasisClient legalBasisClient;
+
+
+    @Before
+    public void stubClients() {
+
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
+
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    @After
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    /**
+     * Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
+     * If the RedactionLog already exists, the generating will be skipped
+     * Test is ignored, because it's for manual tests.
+     */
+    @Ignore
+    @Test
+    public void generateRedactionLogForOneFile() {
+
+        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
+        generateAndSaveRedactionLog(fileName);
+    }
+
+
+    /**
+     * Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
+     * If a RedactionLog already exists, the generating will be skipped
+     */
+    @IfProfileValue(name = "test-groups", value = "rules-test")
+    @Test
+    public void generateRedactionLogForAllFiles() {
+
+        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
+        System.out.println("Will generate RedactionLog for " + files.size() + " files.");
+        files.forEach(this::generateAndSaveRedactionLog);
+    }
+
+
+    /**
+     * Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
+     * If RedactionLog Json does not exist, test will fail.
+     * Test is ignored, because it's for manual tests.
+     */
+    @Ignore
+    @Test
+    public void analyseFileAndCompareRedactionLog() {
+
+        String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
+        analyseFileAndCompareRedactionLog(fileName);
+    }
+
+
+    /**
+     * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
+     * If RedactionLogs Json does not exist, test will fail.
+     */
+    @IfProfileValue(name = "test-groups", value = "rules-test")
+    @Test
+    public void analyseAllFilesAndCompareRedactionLogs() {
+
+        Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
+        System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
+        files.forEach(this::analyseFileAndCompareRedactionLog);
+    }
+
+
+    @SneakyThrows
+    private void generateAndSaveRedactionLog(String fileName) {
+
+        if (doesRedactionLogExist(fileName)) {
+            System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
+
+        } else {
+            generateTestFileId(fileName);
+
+            System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
+
+            loadNerForTest();
+
+            AnalyzeRequest request = prepareStorage(fileName);
+            analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+            analyzeService.analyze(request);
+
+            RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+            saveRedactionLogAsJson(redactionLog, fileName);
+
+        }
+    }
+
+
+    @SneakyThrows
+    public void analyseFileAndCompareRedactionLog(String fileName) {
+
+        generateTestFileId(fileName);
+        System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
+
+        RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
+
+        loadNerForTest();
+
+        AnalyzeRequest request = prepareStorage(fileName);
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        // All timestamps are ignored, because they are for sure different
+        assertThat(redactionLog.getAnalysisVersion()).isEqualTo(savedRedactionLog.getAnalysisVersion());
+        assertThat(redactionLog.getAnalysisNumber()).isEqualTo(savedRedactionLog.getAnalysisNumber());
+        assertThat(redactionLog.getDictionaryVersion()).isEqualTo(savedRedactionLog.getDictionaryVersion());
+        assertThat(redactionLog.getDossierDictionaryVersion()).isEqualTo(savedRedactionLog.getDossierDictionaryVersion());
+        assertThat(redactionLog.getRulesVersion()).isEqualTo(savedRedactionLog.getRulesVersion());
+        assertThat(redactionLog.getLegalBasisVersion()).isEqualTo(savedRedactionLog.getLegalBasisVersion());
+
+        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(savedRedactionLog.getRedactionLogEntry().size());
+        assertThat(redactionLog.getLegalBasis().size()).isEqualTo(savedRedactionLog.getLegalBasis().size());
+
+        for (RedactionLogLegalBasis redactionLegalBasis : redactionLog.getLegalBasis()) {
+            var savedRedactionLegalBasis = savedRedactionLog.getLegalBasis()
+                    .stream()
+                    .filter(lb -> lb.getName().equalsIgnoreCase(redactionLegalBasis.getName()))
+                    .filter(lb -> lb.getDescription().equalsIgnoreCase(redactionLegalBasis.getDescription()))
+                    .filter(lb -> lb.getReason().equalsIgnoreCase(redactionLegalBasis.getReason()))
+                    .findFirst();
+            assertThat(savedRedactionLegalBasis).isPresent();
+        }
+
+        for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
+            var savedRedactionLogEntry = savedRedactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findFirst();
+            assertThat(savedRedactionLogEntry).isPresent();
+            assertThat(savedRedactionLogEntry.get().getId()).isEqualTo(redactionLogEntry.getId());
+            assertThat(savedRedactionLogEntry.get().getType()).isEqualTo(redactionLogEntry.getType());
+            assertThat(savedRedactionLogEntry.get().getValue()).isEqualTo(redactionLogEntry.getValue());
+            assertThat(savedRedactionLogEntry.get().getReason()).isEqualTo(redactionLogEntry.getReason());
+            assertThat(savedRedactionLogEntry.get().getMatchedRule()).isEqualTo(redactionLogEntry.getMatchedRule());
+            assertThat(savedRedactionLogEntry.get().isRectangle()).isEqualTo(redactionLogEntry.isRectangle());
+            assertThat(savedRedactionLogEntry.get().getLegalBasis()).isEqualTo(redactionLogEntry.getLegalBasis());
+            assertThat(savedRedactionLogEntry.get().isImported()).isEqualTo(redactionLogEntry.isImported());
+            assertThat(savedRedactionLogEntry.get().isRedacted()).isEqualTo(redactionLogEntry.isRedacted());
+            assertThat(savedRedactionLogEntry.get().isHint()).isEqualTo(redactionLogEntry.isHint());
+            assertThat(savedRedactionLogEntry.get().isRecommendation()).isEqualTo(redactionLogEntry.isRecommendation());
+            assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.isFalsePositive());
+            assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
+            assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
+            assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
+            assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
+            assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
+            assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
+            assertThat(savedRedactionLogEntry.get().getEndOffset()).isEqualTo(redactionLogEntry.getEndOffset());
+            assertThat(savedRedactionLogEntry.get().isImage()).isEqualTo(redactionLogEntry.isImage());
+            assertThat(savedRedactionLogEntry.get().isImageHasTransparency()).isEqualTo(redactionLogEntry.isImageHasTransparency());
+            assertThat(savedRedactionLogEntry.get().isDictionaryEntry()).isEqualTo(redactionLogEntry.isDictionaryEntry());
+            assertThat(savedRedactionLogEntry.get().isDossierDictionaryEntry()).isEqualTo(redactionLogEntry.isDossierDictionaryEntry());
+            assertThat(savedRedactionLogEntry.get().isExcluded()).isEqualTo(redactionLogEntry.isExcluded());
+            assertThat(savedRedactionLogEntry.get().getSourceId()).isEqualTo(redactionLogEntry.getSourceId());
+
+            for (Rectangle rectangle : redactionLogEntry.getPositions()) {
+                var savedRectangle = savedRedactionLogEntry.get()
+                        .getPositions()
+                        .stream()
+                        .filter(r -> r.getPage() == rectangle.getPage())
+                        .filter(r -> r.getTopLeft().getX() == rectangle.getTopLeft().getX())
+                        .filter(r -> r.getTopLeft().getY() == rectangle.getTopLeft().getY())
+                        .filter(r -> r.getHeight() == rectangle.getHeight())
+                        .filter(r -> r.getWidth() == rectangle.getWidth())
+                        .findFirst();
+                assertThat(savedRectangle).isPresent();
+            }
+
+            for (RedactionLogComment comment : redactionLogEntry.getComments()) {
+                var savedComment = savedRedactionLogEntry.get().getComments().stream().filter(c -> c.getId() == comment.getId()).findFirst();
+                assertThat(savedComment).isPresent();
+                assertThat(savedComment.get().getId()).isEqualTo(comment.getId());
+                assertThat(savedComment.get().getUser()).isEqualTo(comment.getUser());
+                assertThat(savedComment.get().getText()).isEqualTo(comment.getText());
+                assertThat(savedComment.get().getAnnotationId()).isEqualTo(comment.getAnnotationId());
+                assertThat(savedComment.get().getFileId()).isEqualTo(comment.getFileId());
+
+            }
+
+            for (Change change : redactionLogEntry.getChanges()) {
+                var savedChange = savedRedactionLogEntry.get()
+                        .getChanges()
+                        .stream()
+                        .filter(c -> c.getAnalysisNumber() == change.getAnalysisNumber())
+                        .filter(c -> c.getType() == change.getType())
+                        .findFirst();
+                assertThat(savedChange).isPresent();
+            }
+
+            for (ManualChange manualChange : redactionLogEntry.getManualChanges()) {
+                var savedManualChange = savedRedactionLogEntry.get()
+                        .getManualChanges()
+                        .stream()
+                        .filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
+                        .filter(m -> m.getManualRedactionType() == manualChange.getManualRedactionType())
+                        .filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
+                        .filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
+                        .findFirst();
+                assertThat(savedManualChange).isPresent();
+            }
+
+            assertThat(savedRedactionLogEntry.get().getEngines()).containsExactly(redactionLogEntry.getEngines().toArray(new Engine[0]));
+
+            assertThat(savedRedactionLogEntry.get().getReference()).containsAll(redactionLogEntry.getReference());
+            assertThat(savedRedactionLogEntry.get().getImportedRedactionIntersections()).containsAll(redactionLogEntry.getImportedRedactionIntersections());
+        }
+
+    }
+
+
+    private boolean doesRedactionLogExist(String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+        return file.exists();
+
+    }
+
+
+    @SneakyThrows
+    private void generateTestFileId(String fileName) {
+
+        MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
+        messageDigest.update(fileName.getBytes());
+        TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
+    }
+
+
+    @SneakyThrows
+    private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        File dr = new File(directory);
+        boolean created = dr.mkdirs();
+        if (created) {
+            System.out.println("Directory was created");
+        }
+
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+
+        objectMapper.writeValue(file, redactionLog);
+
+        System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
+    }
+
+
+    @SneakyThrows
+    private RedactionLog loadSavedRedactionLog(String pdfFileName) {
+
+        File pdfFile = new File(pdfFileName);
+        String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
+        String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
+        File file = new File(directory, fileName);
+        assertThat(file).exists();
+
+        return objectMapper.readValue(file, RedactionLog.class);
+    }
+
+
+    @SneakyThrows
+    private void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+    }
+
+
+    @SneakyThrows
+    private AnalyzeRequest prepareStorage(String file) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+
+        return prepareStorage(pdfFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    private AnalyzeRequest prepareStorage(InputStream stream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
+
+        return request;
+
+    }
+
+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
+
+        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+    }
+
+
+    private void loadTypeForTest() {
+
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
+    }
+
+
+    private List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+
+    private void mockDictionaryCalls(Long version) {
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
+
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = path.toAbsolutePath().toString();
+                    int pos = StringUtils.indexOf(absolutePath, StringUtils.replace(RESOURCES_PATH, "/", "\\")) + 18;
+                    fileNames.add(StringUtils.substring(absolutePath, pos));
+                }
+            }
+        }
+        return fileNames;
+    }
+
+
+    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
+                .falseRecommendationEntries(new ArrayList<>())
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(0L).deleted(false).build()));
+        return dictionaryEntries;
+    }
+
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class RulesTestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write(RESOURCES_PATH + RULES_PATH, kieServices.getResources().newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+}
@@ -1,22 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.realdata;

+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.nullable;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;

-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.*;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -35,13 +30,27 @@ import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
 import org.springframework.test.context.junit4.SpringRunner;

-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.when;
+import lombok.SneakyThrows;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -114,9 +123,8 @@ public class LiveDataIntegrationTest {
        ObjectMapper objectMapper = new ObjectMapper();

        var jsonNode = objectMapper.readTree(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "types.json").getInputStream());
-        types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(
-                new TypeReference<List<Type>>() {
-                }));
+        types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(new TypeReference<List<Type>>() {
+        }));

        types.forEach(t -> {
            t.setId(t.getType());
@@ -126,27 +134,28 @@ public class LiveDataIntegrationTest {
        when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(types);
        when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>());

-
        when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(), Colors.class));

        when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
            String typeName = answer.getArgument(0);

            var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
-            if(found.isPresent()) {
+            if (found.isPresent()) {
                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
                type.setEntries(getEntries(typeName, type.getTypeId()));

                return type;
-            }else{
+            } else {
                return null;
            }
        });

-        dictionaryService.updateDictionary("dossierTemplateId","dossierId");
+        dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
    }

+
    public void simulateIncrement(List<String> values, String deltaTypeName, long version) {
+
        when(dictionaryClient.getVersion(anyString())).thenReturn(version);
        when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(version);

@@ -154,7 +163,7 @@ public class LiveDataIntegrationTest {
            String typeName = answer.getArgument(0);

            var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
-            if(found.isPresent()) {
+            if (found.isPresent()) {

                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();

@@ -165,7 +174,7 @@ public class LiveDataIntegrationTest {
                }

                return type;
-            }else{
+            } else {
                return null;
            }
        });
@@ -174,17 +183,20 @@ public class LiveDataIntegrationTest {

    @Test
    public void testUpdateDictionary() {
+
        dictionaryService.updateDictionary("dossierTemplateId", "dossierId");

        var dict = dictionaryService.getDeepCopyDictionary("dossierTemplateId", "dossierId");
        assertThat(dict.getLocalAccessMap().size()).isEqualTo(12);
    }

+
    @SneakyThrows
    private List<DictionaryEntry> getEntries(String typeName, String typeId) {
+
        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
        for (var resource : dictionaryResources) {
-            if (resource.getFilename().contains(typeName)) {
+            if (Objects.requireNonNull(resource.getFilename()).contains(typeName)) {

                List<String> lines = IOUtils.readLines(resource.getInputStream());
                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
@@ -193,4 +205,5 @@ public class LiveDataIntegrationTest {
        }
        return new ArrayList<>();
    }
+
 }
@@ -1,23 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.segmentation;

-import com.amazonaws.services.s3.AmazonS3;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
-import org.junit.Ignore;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 import org.junit.Test;
-import org.junit.platform.commons.util.StringUtils;
 import org.junit.runner.RunWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
@@ -31,20 +25,20 @@ import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;

-import javax.imageio.ImageIO;
-import java.io.ByteArrayOutputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils.getTemporaryDirectory;
-import static org.assertj.core.api.Assertions.assertThat;
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;

 import lombok.SneakyThrows;

@@ -89,15 +83,18 @@ public class PdfSegmentationServiceTest {

    @Test
    @SneakyThrows
-    public void testMapping(){
+    public void testMapping() {
+
        ClassPathResource responseJson = new ClassPathResource("files/image_response.json");
        ImageServiceResponse imageServiceResponse = objectMapper.readValue(responseJson.getInputStream(), ImageServiceResponse.class);

        Map<Integer, List<PdfImage>> images = new HashMap<>();
-        imageServiceResponse.getData().stream().forEach(imageMetadata -> {
-            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
-                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)), imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber()));
-        });
+        imageServiceResponse.getData()
+                .forEach(imageMetadata -> images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
+                        .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry()
+                                .getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification()
+                                .getLabel()
+                                .toUpperCase(Locale.ROOT)), imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber())));

        System.out.println("object");
    }
@@ -1,13 +1,14 @@
 {
  "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
-  "defaultColor": "#9398a0",
-  "requestAdd": "#04b093",
-  "requestRemove": "#04b093",
-  "notRedacted": "#c498fa",
+  "requestAddColor": "#04b093",
+  "requestRemoveColor": "#04b093",
  "analysisColor": "#dd4d50",
  "updatedColor": "#fdbd00",
  "dictionaryRequestColor": "#5b97db",
-  "manualRedactionColor": "#9398a0",
  "previewColor": "#9398a0",
-  "ignoredHintColor": "#e7d4ff"
-}
+  "ignoredHintColor": "#e7d4ff",
+  "recommendationColor": "#cccccc",
+  "hintColor": "#cccccc",
+  "redactionColor": "#cccccc",
+  "skippedColor": "#cccccc"
+}
@@ -12,4 +12,5 @@ Xinyi Y. Tao
 Dorn
 Prasher
 David
-annotation
+annotation
+J.B. RASCLE
@@ -0,0 +1,431 @@
+package drools
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+
+global Section section
+
+
+// --------------------------------------- AI rules -------------------------------------------------------------------
+
+rule "0: Add CBI_author from ai"
+    when
+        Section(aiMatchesType("CBI_author"))
+    then
+        section.addAiEntities("CBI_author", "CBI_author");
+    end
+
+rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
+    when
+        Section(aiMatchesType("ORG"))
+    then
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
+    when
+        Section(aiMatchesType("STREET"))
+    then
+        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
+    when
+        Section(aiMatchesType("CITY"))
+    then
+        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
+    end
+
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
+
+rule "1: Redact CBI Authors (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "2: Redact CBI Authors (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "3: Redact not CBI Address (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        section.ignoreRecommendations("CBI_address");
+    end
+
+rule "4: Redact CBI Address (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "5: Do not redact genitive CBI_author"
+    when
+        Section(matchesType("CBI_author"))
+    then
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+    end
+
+
+rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+/* Syngenta specific laboratory rule */
+rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
+    when
+        Section(searchText.contains("CT") || searchText.contains("BL"))
+    then
+        section.addRecommendationByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address");
+    end
+
+rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "16: Add recommendation for Addresses in Test Organism sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
+    then
+        section.recommendLineAfter("Source:", "CBI_address");
+    end
+
+rule "17: Add recommendation for Addresses in Test Animals sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
+    then
+        section.recommendLineAfter("Source", "CBI_address");
+    end
+
+
+rule "18: Do not redact Names and Addresses if Published Information found"
+    when
+        Section(matchesType("published_information"))
+    then
+        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
+        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
+    end
+
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+
+rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "21: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "22: Redact Emails by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "23: Redact contact information (Non vertebrate study)"
+    when
+       Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+            || text.contains("Phone:")
+            || text.contains("Fax:")
+            || text.contains("Tel.:")
+            || text.contains("Tel:")
+            || text.contains("E-mail:")
+            || text.contains("Email:")
+            || text.contains("e-mail:")
+            || text.contains("E-mail address:")
+            || text.contains("Contact:")
+            || text.contains("Alternative contact:")
+            || text.contains("Telephone number:")
+            || text.contains("Telephone No:")
+            || text.contains("Fax number:")
+            || text.contains("Telephone:")
+            || text.contains("Phone No.")
+            || (text.contains("No:") && text.contains("Fax"))
+            || (text.contains("Contact:") && text.contains("Tel.:"))
+            || text.contains("European contact:")
+        ))
+    then
+        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+     end
+
+rule "24: Redact contact information (Vertebrate study)"
+    when
+       Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+            || text.contains("Phone:")
+            || text.contains("Fax:")
+            || text.contains("Tel.:")
+            || text.contains("Tel:")
+            || text.contains("E-mail:")
+            || text.contains("Email:")
+            || text.contains("e-mail:")
+            || text.contains("E-mail address:")
+            || text.contains("Contact:")
+            || text.contains("Alternative contact:")
+            || text.contains("Telephone number:")
+            || text.contains("Telephone No:")
+            || text.contains("Fax number:")
+            || text.contains("Telephone:")
+            || text.contains("Phone No.")
+            || (text.contains("No:") && text.contains("Fax"))
+            || (text.contains("Contact:") && text.contains("Tel.:"))
+            || text.contains("European contact:")
+        ))
+    then
+        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+     end
+
+
+rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+               text.contains("Telephone")
+            || text.contains("Phone")
+            || text.contains("Ph.")
+            || text.contains("Fax")
+            || text.contains("Tel")
+            || text.contains("Ter")
+            || text.contains("Cell")
+            || text.contains("Mobile")
+            || text.contains("Fel")
+            || text.contains("Fer")
+        ))
+    then
+        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+               text.contains("Telephone")
+            || text.contains("Phone")
+            || text.contains("Ph.")
+            || text.contains("Fax")
+            || text.contains("Tel")
+            || text.contains("Ter")
+            || text.contains("Cell")
+            || text.contains("Mobile")
+            || text.contains("Fel")
+            || text.contains("Fer")
+        ))
+    then
+        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "27: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("COMPLETION DATE:")
+            && !searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "28: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("COMPLETION DATE:")
+            && !searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "29: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "30: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("PERFORMING LABORATORY:")
+        )
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
+    end
+
+rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("PERFORMING LABORATORY:"))
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "33: Purity Hint"
+    when
+        Section(searchText.toLowerCase().contains("purity"))
+    then
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+rule "34: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction"));
+    then
+        section.ignore("dossier_redaction");
+    end
+
+
+rule "35: Redact signatures (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "36: Redact signatures (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "43: Redact Logos (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
+    then
+        section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
@@ -1,12 +1,14 @@
 {
-  "defaultColor": "#9398a0",
-  "requestAdd": "#04b093",
-  "requestRemove": "#04b093",
-  "notRedacted": "#c498fa",
+  "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+  "requestAddColor": "#04b093",
+  "requestRemoveColor": "#04b093",
  "analysisColor": "#dd4d50",
  "updatedColor": "#fdbd00",
  "dictionaryRequestColor": "#5b97db",
-  "manualRedactionColor": "#9398a0",
  "previewColor": "#9398a0",
-  "ignoredHintColor": "#e7d4ff"
+  "ignoredHintColor": "#e7d4ff",
+  "recommendationColor": "#cccccc",
+  "hintColor": "#cccccc",
+  "redactionColor": "#cccccc",
+  "skippedColor": "#cccccc"
 }
Author	SHA1	Message	Date
Timo Bejan	9c8e2f047e	Pull request #432 : RED-4686 - storage commons update Merge in RED/redaction-service from RED-4686 to master * commit '24612e347171f27eb8747772b82d13c98f40344b': RED-4686 - storage commons update - added annotations and test-file RED-4686 - storage commons update	2022-07-25 21:19:43 +02:00
Timo Bejan	24612e3471	RED-4686 - storage commons update - added annotations and test-file	2022-07-25 22:08:36 +03:00
Timo Bejan	9be8d46f4e	RED-4686 - storage commons update	2022-07-25 21:45:58 +03:00
Philipp Schramm	3e32540616	Pull request #431 : RED-4771: Reformatted code Merge in RED/redaction-service from RED-4771 to master * commit '5403829fd874c04375bef39535c22f52f7a55853': RED-4771: Reformatted code	2022-07-25 16:11:37 +02:00
Philipp Schramm	5403829fd8	Merge branch 'master' into RED-4771 # Conflicts: # bamboo-specs/src/main/java/buildjob/PlanSpec.java	2022-07-25 16:08:23 +02:00
Philipp Schramm	3e53e156a5	RED-4771: Reformatted code	2022-07-25 16:04:51 +02:00
Philipp Schramm	4c73d2841b	Pull request #430 : RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist Merge in RED/redaction-service from RED-4510 to master * commit 'db8eedc9a36dfba22946940b31e28854ffdca26d': RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist	2022-07-25 14:52:53 +02:00
Philipp Schramm	db8eedc9a3	RED-4510: Added bamboo build to execute tests nightly and generate RedactionLog for tests if it doesn't exist	2022-07-25 14:14:44 +02:00
Dominique Eiflaender	58e16fe39a	Pull request #429 : RED-4686: Removed test scope from jackson-datatype-jsr310 Merge in RED/redaction-service from RED-4686-2 to master * commit '0fe2b627f071f6d8e4841cf08e3bcdc059b60567': RED-4686: Removed test scope from jackson-datatype-jsr310	2022-07-22 15:10:33 +02:00
deiflaender	0fe2b627f0	RED-4686: Removed test scope from jackson-datatype-jsr310	2022-07-22 15:06:57 +02:00
Dominique Eiflaender	e4b5bb93e1	Pull request #426 : RED-4686: Stream objects to storage Merge in RED/redaction-service from RED-4686 to master * commit '99a8b9788e1b1062c8d8a93d4b5d4b65f3d34ce2': RED-4686: Stream objects to storage	2022-07-22 13:54:23 +02:00
deiflaender	99a8b9788e	RED-4686: Stream objects to storage	2022-07-22 13:29:22 +02:00
Philipp Schramm	822e1f096f	Pull request #425 : RED-4510 Merge in RED/redaction-service from RED-4510 to master * commit '8c5ede3fde0fd1b2df9e8fca0908e9233d6d403c': RED-4510: Refactoring after review RED-4510: Implemented Test class which generates and compares RedactionLogs	2022-07-21 17:01:18 +02:00
Philipp Schramm	8c5ede3fde	RED-4510: Refactoring after review	2022-07-21 16:58:36 +02:00
Philipp Schramm	40380a12a3	Merge branch 'master' into RED-4510	2022-07-21 12:38:27 +02:00
Philipp Schramm	f48bdb6267	RED-4510: Implemented Test class which generates and compares RedactionLogs	2022-07-21 12:38:07 +02:00
Dominique Eiflaender	d3a5265b7f	Pull request #424 : RED-4648: Fixed wrong field names for DSLJson Merge in RED/redaction-service from RED-4648 to master * commit '708b7f992cadba8779ac6d2f7a9e9e4a837eaf1b': RED-4648: Fixed wrong field names for DSLJson	2022-07-21 12:33:42 +02:00
deiflaender	708b7f992c	RED-4648: Fixed wrong field names for DSLJson	2022-07-21 12:22:19 +02:00
Ali Oezyetimoglu	2b0a667424	Pull request #422 : RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service) Merge in RED/redaction-service from RED-4548-rs2 to master * commit 'c330de86246b8cc10955401d307d7d1a20190e32': RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service)	2022-07-14 17:08:03 +02:00
Ali Oezyetimoglu	c330de8624	RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service)	2022-07-14 16:57:09 +02:00
Timo Bejan	32e73dc9d9	Pull request #420 : RED-4593 Merge in RED/redaction-service from tbejan/pomxml-1657693418623 to master * commit '5e61f3370dcec8206b757dd59a1a7664921c8300': RED-4593	2022-07-13 08:32:03 +02:00
Timo Bejan	5e61f3370d	RED-4593	2022-07-13 08:23:59 +02:00
Philipp Schramm	bfadac7a3f	Pull request #419 : RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service) Merge in RED/redaction-service from RED-4548 to master * commit 'df80ff68cd21626679c107b056b275b67ce6bf74': RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service)	2022-07-12 16:41:06 +02:00
Philipp Schramm	df80ff68cd	RED-4548: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service)	2022-07-12 16:38:25 +02:00
Timo Bejan	13352e7d16	Pull request #418 : RED-4525 - Colors update Merge in RED/redaction-service from RED-4525 to master * commit '3c9d59f73ba2ca2b7500703b57df6d78b558d6a2': RED-4525 - Colors update	2022-07-11 20:58:27 +02:00
Timo Bejan	3c9d59f73b	RED-4525 - Colors update	2022-07-11 21:55:29 +03:00
Dominique Eiflaender	333793021a	Pull request #417 : RED-4254: Added dirty hack in pdfbox classes to find words that contains uniqueCharacters with 2 chars like 'RA' Merge in RED/redaction-service from RED-4254 to master * commit '264d7e3a87d4a99aa60a4c7c140335ba8d3a7241': RED-4254: Added dirty hack in pdfbox classes to find words that contains uniqueCharacters with 2 chars like 'RA'	2022-06-28 15:30:02 +02:00
deiflaender	264d7e3a87	RED-4254: Added dirty hack in pdfbox classes to find words that contains uniqueCharacters with 2 chars like 'RA'	2022-06-28 15:11:33 +02:00
Dominique Eiflaender	6b89572c87	Pull request #416 : RED-4407: Fixed merging request for dossierDictionarys in redactionLog Merge in RED/redaction-service from RED-4407 to master * commit 'a74d049cf8a52395771f9940471defd692cae0e8': RED-4407: Fixed merging request for dossierDictionarys in redactionLog	2022-06-28 11:55:12 +02:00
deiflaender	a74d049cf8	RED-4407: Fixed merging request for dossierDictionarys in redactionLog	2022-06-28 11:52:06 +02:00
Ali Oezyetimoglu	76dc08d0e6	Pull request #415 : RED-4350: Reason set to null after reject suggestion "Remove from Dict" for skipped types Merge in RED/redaction-service from RED-4350-rs1 to master * commit '663c361707678370149ed08cb6c33c8ac341dc26': RED-4350: Reason set to null after reject suggestion "Remove from Dict" for skipped types	2022-06-28 09:01:05 +02:00
aoezyetimoglu	663c361707	RED-4350: Reason set to null after reject suggestion "Remove from Dict" for skipped types	2022-06-27 17:43:28 +02:00
Dominique Eiflaender	c81b062784	Pull request #413 : RED-4301: Revert redaction log merge of duplicate annotations Merge in RED/redaction-service from RED-4301 to master * commit '57ef7ead72018a91d9c15530efce40f713fb6c86': RED-4301: Revert redaction log merge of duplicate annotations	2022-06-20 12:50:53 +02:00
deiflaender	57ef7ead72	RED-4301: Revert redaction log merge of duplicate annotations	2022-06-20 12:38:57 +02:00
Dominique Eiflaender	3269470446	Pull request #412 : RED-4129: Fixed imported redactions annotations intersection calculation Merge in RED/redaction-service from RED-4129 to master * commit 'c7e10cefbacf8ca54c86ecba5a8598b7c9d55ce4': RED-4129: Fixed imported redactions annotations intersection calculation	2022-06-20 11:59:58 +02:00
deiflaender	c7e10cefba	RED-4129: Fixed imported redactions annotations intersection calculation	2022-06-20 11:57:10 +02:00
Timo Bejan	2301849815	Pull request #405 : RED-3791 - redaction log merge of duplicate annotations Merge in RED/redaction-service from RED-3791 to master * commit 'ec8d3183dc119e3eddd45a653ca88a2666073f51': RED-3791 - redaction log merge of duplicate annotations	2022-06-14 18:25:35 +02:00
Dominique Eiflaender	2d5ba8f447	Pull request #411 : RED-4129: Fixed calculation of imported redaction intersections for all rotation cases Merge in RED/redaction-service from RED-4129 to master * commit '2a6ee02f9f8f527546f01cf98542dfc7e39b259a': RED-4129: Fixed calculation of imported redaction intersections for all rotation cases	2022-06-14 12:15:01 +02:00
Timo Bejan	ec8d3183dc	RED-3791 - redaction log merge of duplicate annotations	2022-06-09 09:26:00 +03:00