Pull request #385 : RED-3946 - quotation-mark fix

Merge in RED/redaction-service from RED-3946-qf to master * commit 'df2fcd2e5e48298131ebf7a00bf2ed80a9fc2d96': RED-3946 - quotation-mark fix
Pull request #386 : RED-3959: Do not remove entities on larger false_recommendactions
2022-05-09 14:47:03 +02:00 · 2022-05-09 12:34:13 +02:00 · 2022-05-09 12:27:36 +02:00 · 2022-05-09 12:14:09 +03:00 · 2022-05-09 10:05:50 +02:00 · 2022-05-09 09:02:11 +03:00
93 changed files with 44803 additions and 2022 deletions
@@ -5,7 +5,7 @@
  <parent>
    <groupId>com.atlassian.bamboo</groupId>
    <artifactId>bamboo-specs-parent</artifactId>
-    <version>8.0.3</version>
+    <version>8.1.3</version>
    <relativePath/>
  </parent>

@@ -17,6 +17,7 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
+import com.atlassian.bamboo.specs.api.builders.Variable;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
@@ -83,6 +84,7 @@ public class PlanSpec {
                project(),
                SERVICE_NAME, new BambooKey(SERVICE_KEY))
                .description("Plan created from (enter repository url of your plan)")
+                .variables(new Variable("maven_add_param", ""))
                .stages(new Stage("Default Stage")
                        .jobs(new Job("Default Job",
                                new BambooKey("JOB1"))
@@ -94,6 +96,7 @@ public class PlanSpec {
                                                        "rm -rf ./*"),
                                        new VcsCheckoutTask()
                                                .description("Checkout Default Repository")
+                                                .cleanCheckout(true)
                                                .checkoutItems(new CheckoutItem().defaultRepository()),
                                        new ScriptTask()
                                                .description("Build")
@@ -115,7 +118,7 @@ public class PlanSpec {
                                                .defaultRepository())
                                .dockerConfiguration(
                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
                                )
@@ -147,6 +150,7 @@ public class PlanSpec {
                                                        "rm -rf ./*"),
                                        new VcsCheckoutTask()
                                                .description("Checkout Default Repository")
+                                                .cleanCheckout(true)
                                                .checkoutItems(new CheckoutItem().defaultRepository()),
                                        new ScriptTask()
                                                .description("Sonar")
@@ -155,9 +159,9 @@ public class PlanSpec {
                                                .argument(SERVICE_NAME))
                                .dockerConfiguration(
                                        new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                                .dockerRunArguments("--net=host")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
                                                .volume("/var/run/docker.sock", "/var/run/docker.sock")
                                )
                        )
@@ -165,8 +169,7 @@ public class PlanSpec {
                .linkedRepositories("RED / " + SERVICE_NAME)
                .triggers(
                        new ScheduledTrigger()
-                                .scheduleOnceDaily(LocalTime.of(12, 00)),
-                        new BitbucketServerTrigger())
+                                .scheduleOnceDaily(LocalTime.of(23, 00)))
                .planBranchManagement(new PlanBranchManagement()
                        .createForVcsBranchMatching("release.*")
                        .notificationForCommitters());
@@ -3,49 +3,62 @@ set -e

 SERVICE_NAME=$1

-if [[ "${bamboo_version_tag}" = "dev" ]]
-then 
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+if [[ "$bamboo_planRepository_branchName" == "master" ]]
+then
+    branchVersion=$(cat pom.xml | grep -Eo "    <version>.*-SNAPSHOT</version>" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|' | tr -d ' ')
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
+    newVersion="$(semver $latestVersion -p -i minor)"
+    echo "new release on master with version $newVersion"
+elif [[ "$bamboo_planRepository_branchName" == release* ]]
+then
+    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
+    newVersion="$(semver $latestVersion -p -i patch)"
+    echo "new release on $bamboo_planRepository_branchName with version $newVersion"
+elif [[ "${bamboo_version_tag}" != "dev" ]]
+then
+    newVersion="${bamboo_version_tag}"
+    echo "new special version bild with $newVersion"
+else
+    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
        --no-transfer-progress \
+        ${bamboo_maven_add_param} \
        clean install \
        -Djava.security.egd=file:/dev/./urandomelse
-else        
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        versions:set \
-        -DnewVersion=${bamboo_version_tag}
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-        versions:set  \
-        -DnewVersion=${bamboo_version_tag}
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        --no-transfer-progress \
-        clean deploy \
-        -e \
-        -DdeployAtEnd=true \
-        -Dmaven.wagon.http.ssl.insecure=true \
-        -Dmaven.wagon.http.ssl.allowall=true \
-        -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-        -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+    echo "dev build with tag ${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
+    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
+    exit 0
 fi

-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+echo "gitTag=${newVersion}" > git.tag
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    ${bamboo_maven_add_param} \
+    versions:set \
+    -DnewVersion=${newVersion}
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    ${bamboo_maven_add_param} \
+    versions:set  \
+    -DnewVersion=${newVersion}
+
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
    --no-transfer-progress \
+    clean deploy \
+    ${bamboo_maven_add_param} \
+    -e \
+    -DdeployAtEnd=true \
+    -Dmaven.wagon.http.ssl.insecure=true \
+    -Dmaven.wagon.http.ssl.allowall=true \
+    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
+    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+
+mvn --no-transfer-progress \
    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
    package

-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-    --no-transfer-progress \
+mvn --no-transfer-progress \
    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
    docker:push
-
-if [[ "${bamboo_version_tag}" = "dev" ]]
-then 
-    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
-else
-    echo "gitTag=${bamboo_version_tag}" > git.tag
-fi
@@ -3,46 +3,42 @@ set -e

 SERVICE_NAME=$1

-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+echo "build jar binaries"
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
    --no-transfer-progress \
    clean install \
    -Djava.security.egd=file:/dev/./urandomelse

 echo "dependency-check:aggregate"
-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-    --no-transfer-progress \
+mvn --no-transfer-progress \
    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
    org.owasp:dependency-check-maven:aggregate

 if [[ -z "${bamboo_repository_pr_key}" ]]
 then
    echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-            --no-transfer-progress \
-            -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-            sonar:sonar \
-            -Dsonar.projectKey=RED_$SERVICE_NAME \
-            -Dsonar.host.url=https://sonarqube.iqser.com \
-            -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-            -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-            -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-            -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-            -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-            
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
 else
    echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-            --no-transfer-progress \
-            -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-            sonar:sonar \
-            -Dsonar.projectKey=RED_$SERVICE_NAME \
-            -Dsonar.host.url=https://sonarqube.iqser.com \
-            -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-            -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-            -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-            -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-            -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-            -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-            -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-fi
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
+        -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
+        -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
+fi
@@ -7,7 +7,7 @@

    <artifactId>redaction-service</artifactId>
    <groupId>com.iqser.red.service</groupId>
-    <version>1.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>


    <packaging>pom</packaging>
@@ -18,4 +18,4 @@
        <module>redaction-service-image-v1</module>
    </modules>

-</project>
+</project>
@@ -5,7 +5,7 @@
    <parent>
        <groupId>com.iqser.red</groupId>
        <artifactId>platform-docker-dependency</artifactId>
-        <version>1.1.0</version>
+        <version>1.2.0</version>
        <relativePath />
    </parent>
    <modelVersion>4.0.0</modelVersion>
@@ -42,7 +42,7 @@
                <artifactId>docker-maven-plugin</artifactId>
            </plugin>
        </plugins>
-        
+
        <pluginManagement>
            <plugins>
                <plugin>
@@ -95,4 +95,4 @@
            </plugins>
        </pluginManagement>
    </build>
-</project>
+</project>
@@ -1,4 +1,4 @@
-FROM red/base-image:1.0.0
+FROM red/redaction-service-base-v1:1.0.0

 ARG PLATFORM_JAR

@@ -7,12 +7,3 @@ ENV PLATFORM_JAR ${PLATFORM_JAR}
 ENV USES_ELASTICSEARCH false

 COPY ["${PLATFORM_JAR}", "/"]
-
-RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    wget cabextract xfonts-utils fonts-liberation \
- && rm -rf /var/lib/apt/lists/*
-
-RUN curl http://ftp.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb -o /tmp/ttf-mscorefonts-installer_3.8_all.deb \
- && dpkg -i /tmp/ttf-mscorefonts-installer_3.8_all.deb \
- && rm /tmp/ttf-mscorefonts-installer_3.8_all.deb \
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.6.0</version>
+        <version>1.10.0</version>
        <relativePath />
    </parent>
    <modelVersion>4.0.0</modelVersion>
@@ -32,7 +32,7 @@
            <dependency>
                <groupId>com.iqser.red</groupId>
                <artifactId>platform-commons-dependency</artifactId>
-                <version>1.7.0</version>
+                <version>1.11.0</version>
                <scope>import</scope>
                <type>pom</type>
            </dependency>
@@ -59,7 +59,7 @@
                    <groupId>org.sonarsource.scanner.maven</groupId>
                    <artifactId>sonar-maven-plugin</artifactId>
                    <version>3.9.0.2155</version>
-                    </plugin>                    
+                    </plugin>
                <plugin>
                    <groupId>org.owasp</groupId>
                    <artifactId>dependency-check-maven</artifactId>
@@ -92,7 +92,7 @@
            <plugin>
                <groupId>org.jacoco</groupId>
                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.7</version>
+                <version>0.8.8</version>
                <executions>
                    <execution>
                        <id>prepare-agent</id>
@@ -12,7 +12,7 @@
    <artifactId>redaction-service-api-v1</artifactId>

    <properties>
-        <persistence-service.version>0.151.0</persistence-service.version>
+        <persistence-service.version>1.166.0</persistence-service.version>
    </properties>

    <dependencies>
@@ -25,6 +25,12 @@
            <groupId>com.iqser.red.service</groupId>
            <artifactId>persistence-service-api-v1</artifactId>
            <version>${persistence-service.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>redaction-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
        </dependency>
    </dependencies>
 </project>
@@ -1,17 +1,17 @@
 package com.iqser.red.service.redaction.v1.model;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;

+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;

@Data
@Builder
@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImportedRedaction {
+
+    private String id;
+
+    @Builder.Default
+    private List<Rectangle> positions = new ArrayList<>();
+}
@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ImportedRedactions {
+
+    @Builder.Default
+    private Map<Integer, List<ImportedRedaction>> importedRedactions = new HashMap<>();
+}
@@ -0,0 +1,50 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.time.OffsetDateTime;
+import java.util.HashMap;
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class ManualChange {
+
+    private AnnotationStatus annotationStatus;
+    private ManualRedactionType manualRedactionType;
+    private OffsetDateTime processedDate;
+    private OffsetDateTime requestedDate;
+    private String userId;
+    private Map<String, Object> propertyChanges = new HashMap<>();
+
+    public boolean isProcessed() {
+        return processedDate != null;
+    }
+
+    public static ManualChange from(BaseAnnotation baseAnnotation) {
+        ManualChange manualChange = new ManualChange();
+        manualChange.annotationStatus = baseAnnotation.getStatus();
+        manualChange.processedDate = baseAnnotation.getProcessedDate();
+        manualChange.requestedDate = baseAnnotation.getRequestDate();
+        manualChange.userId = baseAnnotation.getUser();
+        return manualChange;
+    }
+
+    public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
+        this.manualRedactionType = manualRedactionType;
+        return this;
+    }
+
+    public ManualChange withChange(String property, Object value) {
+        this.propertyChanges.put(property, value);
+        return this;
+    }
+
+}
@@ -1,5 +1,13 @@
 package com.iqser.red.service.redaction.v1.model;

 public enum ManualRedactionType {
-    ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE, RESIZE
+    ADD_LOCALLY,
+    ADD_TO_DICTIONARY,
+    REMOVE_LOCALLY,
+    REMOVE_FROM_DICTIONARY,
+    FORCE_REDACT,
+    FORCE_HINT,
+    RECATEGORIZE,
+    LEGAL_BASIS_CHANGE,
+    RESIZE
 }
@@ -2,6 +2,6 @@ package com.iqser.red.service.redaction.v1.model;

 public enum MessageType {

-    FULL_ANALYSE, REANALYSE, SURROUNDING_TEXT
+    ANALYSE, REANALYSE, STRUCTURE_ANALYSE, SURROUNDING_TEXT

 }
@@ -2,16 +2,9 @@ package com.iqser.red.service.redaction.v1.model;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
+import lombok.*;

-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;


@Data
@@ -28,15 +21,13 @@ public class RedactionLogEntry {
    private int matchedRule;
    private boolean rectangle;
    private String legalBasis;
+
+    private boolean imported;
+
    private boolean redacted;
    private boolean isHint;
    private boolean isRecommendation;
-
-    private boolean hasBeenResized;
-    private boolean hasBeenRecategorized;
-    private boolean hasLegalBasisChanged;
-    private boolean hasBeenForced;
-    private boolean hasBeenRemovedByManualOverride;
+    private boolean isFalsePositive;

    private String section;
    private float[] color;
@@ -44,11 +35,7 @@ public class RedactionLogEntry {
    @Builder.Default
    private List<Rectangle> positions = new ArrayList<>();
    private int sectionNumber;
-    private boolean manual;
-    private AnnotationStatus status;
-    private ManualRedactionType manualRedactionType;
-    private String manualRedactionUserId;
-    private boolean isDictionaryEntry;
+

    private String textBefore;
    private String textAfter;
@@ -62,21 +49,44 @@ public class RedactionLogEntry {
    private boolean isImage;
    private boolean imageHasTransparency;

+    private boolean isDictionaryEntry;
    private boolean isDossierDictionaryEntry;

    private boolean excluded;

-    private String recategorizationType;
-    private String legalBasisChangeValue;
+    private String sourceId;

    @EqualsAndHashCode.Exclude
    @Builder.Default
    private List<Change> changes = new ArrayList<>();

-    private Set<Engine> engines= new HashSet<>();
+    @EqualsAndHashCode.Exclude
+    @Builder.Default
+    private List<ManualChange> manualChanges = new ArrayList<>();
+
+    private Set<Engine> engines = new HashSet<>();

    private Set<String> reference = new HashSet<>();

+    @Builder.Default
+    private Set<String> importedRedactionIntersections = new HashSet<>();

+    public boolean lastChangeIsRemoved() {
+        return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
+    }
+
+    public boolean isLocalManualRedaction() {
+        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY &&
+                mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
+    }
+
+    public boolean isManuallyRemoved() {
+        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY &&
+                mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
+    }
+
+    private <T> Optional<T> last(List<T> list) {
+        return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
+    }

 }
@@ -1,10 +1,15 @@
 package com.iqser.red.service.redaction.v1.model;

 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;


 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+
+
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -22,4 +27,9 @@ public class RedactionRequest {
    private ManualRedactions manualRedactions;
    @Builder.Default
    private Set<Integer> excludedPages = new HashSet<>();
+
+    private Colors colors;
+    private List<Type> types;
+
+    private boolean includeFalsePositives;
 }
@@ -11,10 +11,6 @@ import org.springframework.web.bind.annotation.RequestParam;

 public interface RedactionResource {

-    @PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
-
-
    @PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
    RedactionResult classify(@RequestBody RedactionRequest redactionRequest);

@@ -24,6 +24,19 @@
            <groupId>com.iqser.red.commons</groupId>
            <artifactId>storage-commons</artifactId>
        </dependency>
+
+        <dependency>
+            <groupId>org.ahocorasick</groupId>
+            <artifactId>ahocorasick</artifactId>
+            <version>0.6.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openjdk.jol</groupId>
+            <artifactId>jol-core</artifactId>
+            <version>0.10</version>
+        </dependency>
+
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>redaction-service-api-v1</artifactId>
@@ -52,7 +52,7 @@ public class ClassificationService {
                              List<Float> headlineFontSizes) {

        if (document.getFontSizeCounter().getMostPopular() == null) {
-            // TODO Figure out why this happens.
+            textBlock.setClassification("Other");
            return;
        }
        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
@@ -16,6 +16,6 @@ import lombok.NoArgsConstructor;
 public class NerEntities {

   @Builder.Default
-   private Map<Integer, List<EntityRecogintionEntity>> result = new HashMap<>();
+   private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();

 }
@@ -1,42 +1,29 @@
 package com.iqser.red.service.redaction.v1.server.controller;

-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.io.MemoryUsageSetting;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RestController;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.*;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.stream.Collectors;

@Slf4j
@RestController
@@ -45,53 +32,19 @@ public class RedactionController implements RedactionResource {

    private final PdfVisualisationService pdfVisualisationService;
    private final DroolsExecutionService droolsExecutionService;
-    private final DictionaryService dictionaryService;
-    private final AnnotationService annotationService;
    private final PdfSegmentationService pdfSegmentationService;
    private final RedactionStorageService redactionStorageService;
    private final RedactionLogMergeService redactionLogMergeService;
    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;

-
-    public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest
-                .getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
-        var mergedRedactionLog = getRedactionLog(RedactionRequest.builder()
-                .fileId(annotateRequest.getFileId())
-                .manualRedactions(annotateRequest.getManualRedactions())
-                .dossierId(annotateRequest.getDossierId())
-                .dossierTemplateId(annotateRequest.getDossierTemplateId())
-                .build());
-        var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
-
-        try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
-            pdDocument.setAllSecurityToBeRemoved(true);
-
-            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
-            annotationService.annotate(pdDocument, mergedRedactionLog, sectionsGrid);
-
-            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
-                pdDocument.save(byteArrayOutputStream);
-                return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
-            }
-
-        } catch (Exception e) {
-            throw new RedactionException(e);
-        }
-    }
-
-
    @Override
    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {

-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
        try {
            Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);

-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                    .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
                pdDocument.setAllSecurityToBeRemoved(true);

@@ -113,13 +66,11 @@ public class RedactionController implements RedactionResource {
    @Override
    public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {

-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
        try {
            Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);

-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                    .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
                pdDocument.setAllSecurityToBeRemoved(true);

@@ -143,8 +94,7 @@ public class RedactionController implements RedactionResource {
        Document classifiedDoc;

        try {
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
-                    .getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
+            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
            classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
        } catch (Exception e) {
            throw new RedactionException(e);
@@ -175,44 +125,8 @@ public class RedactionController implements RedactionResource {
    @Override
    public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {

-        log.info("Requested preview for: {}", redactionRequest);
-        dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());

-        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-        if (redactionLog == null) {
-            throw new NotFoundException("RedactionLog not present");
-        }
-
-        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-
-        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest
-                .getFileId());
-        if (sectionGrid.getSections().isEmpty()) {
-
-            log.info("SectionGrid does not have headlines set. Computing headlines now!");
-            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-            // enhance section grid with headline data
-            for (var sectionText : text.getSectionTexts()) {
-                sectionGrid.getSections()
-                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText
-                                .getSectionAreas()
-                                .stream()
-                                .map(SectionArea::getPage)
-                                .collect(Collectors.toSet()), sectionText.getSectionAreas()));
-            }
-            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
-        }
-
-        log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        if (redactionLog.getAnalysisVersion() == 0) {
-            // old redaction logs are returned directly
-            return redactionLog;
-        } else {
-            return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest
-                    .getManualRedactions(), redactionRequest.getExcludedPages());
-        }
+        return redactionLogMergeService.provideRedactionLog(redactionRequest);
    }


@@ -234,11 +148,9 @@ public class RedactionController implements RedactionResource {
                                               @PathVariable("fileId") String fileId,
                                               @RequestBody ManualRedactions manualRedactions) {

-        long start = System.currentTimeMillis();
        var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
-        log.info("add surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, System
-                .currentTimeMillis() - start);
-        return result;
+        log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
+        return result.getManualRedactions();
    }

 }
@@ -240,7 +240,7 @@ public class TextPositionSequence implements CharSequence {
    @JsonIgnore
    public Rectangle getRectangle() {

-        log.debug("Page: '{}', Word: '{}', Rotation: '{}'", page, toString(), textPositions.get(0).getRotation());
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0).getRotation(), textPositions.get(0).getDir());

        float height = getTextHeight();

@@ -291,6 +291,14 @@ public class TextPositionSequence implements CharSequence {
            posXInit = textPositions.get(0).getYDirAdj() + 2;
            posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;

+        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
+            posXInit = textPositions.get(0).getPageWidth() - getX1() +1;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
+                    .getWidthDirAdj() - 3;
+            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
+            posYEnd =  textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() - height + 2;
+
        } else {
            posXEnd = textPositions.get(textPositions.size() - 1)
                    .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class MessageReceiver {
+
+    private final RedactionMessageReceiver redactionMessageReceiver;
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_QUEUE)
+    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+        redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+    }
+
+}
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.queue;

 import lombok.RequiredArgsConstructor;
+
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;

@@ -10,11 +12,24 @@ import org.springframework.context.annotation.Configuration;
@RequiredArgsConstructor
 public class MessagingConfiguration {

-
    public static final String REDACTION_QUEUE = "redactionQueue";
-
    public static final String REDACTION_DQL = "redactionDQL";

+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
+    public MessageReceiver messageReceiver(RedactionMessageReceiver redactionMessageReceiver){
+        return new MessageReceiver(redactionMessageReceiver);
+    }
+
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
+    public PriorityMessageReceiver priorityMessageReceiver(RedactionMessageReceiver redactionMessageReceiver){
+        return new PriorityMessageReceiver(redactionMessageReceiver);
+    }
+

    @Bean
    public Queue redactionQueue() {
@@ -27,9 +42,21 @@ public class MessagingConfiguration {
    }


+    @Bean
+    public Queue redactionPriorityQueue() {
+
+        return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
+                .maxPriority(2)
+                .build();
+    }
+
+
    @Bean
    public Queue redactionDeadLetterQueue() {

        return QueueBuilder.durable(REDACTION_DQL).build();
    }
+
 }
@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class PriorityMessageReceiver {
+
+    private final RedactionMessageReceiver redactionMessageReceiver;
+
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_PRIORITY_QUEUE)
+    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+
+        redactionMessageReceiver.receiveAnalyzeRequest(in, true);
+    }
+
+}
@@ -8,17 +8,13 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.NerAnalyserService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.stereotype.Service;

 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;

@Slf4j
@Service
@@ -28,48 +24,46 @@ public class RedactionMessageReceiver {
    private final ObjectMapper objectMapper;
    private final AnalyzeService analyzeService;
    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
-    private final NerAnalyserService nerAnalyserService;
    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;


-    @RabbitHandler
-    @RabbitListener(queues = REDACTION_QUEUE)
-    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+    public void receiveAnalyzeRequest(String in, boolean priority) throws JsonProcessingException {

        var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
-        log.info("Processing analyze request for file: {}", analyzeRequest.getFileId());
+        log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
        AnalyzeResult result = null;

-        switch (analyzeRequest.getMessageType()){
+        try {
+            switch (analyzeRequest.getMessageType()) {

-            case REANALYSE:
-                result = analyzeService.reanalyze(analyzeRequest);
-                log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
-            case FULL_ANALYSE:
-                // TODO Seperate stucture analysis by other queue
-                analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+                case REANALYSE:
+                    result = analyzeService.reanalyze(analyzeRequest);
+                    log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;

-                // TODO NerEntities should be computed and stored in entity-recognition-service, should be triggered by a seperate queue after structure analysis
-                nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                case STRUCTURE_ANALYSE:
+                    result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+                    log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;

-                result = analyzeService.analyze(analyzeRequest);
-                log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result
-                        .getDuration());
+                case ANALYSE:
+                    result = analyzeService.analyze(analyzeRequest);
+                    log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;

-                break;
-            case SURROUNDING_TEXT:
-                var manualRedactions = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
-                result = AnalyzeResult.builder()
-                        .dossierId(analyzeRequest.getDossierId())
-                        .fileId(analyzeRequest.getFileId())
-                        .manualRedactions(manualRedactions)
-                        .build();
-                break;
+                case SURROUNDING_TEXT:
+                    result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
+                    log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
+            }
+
+            result.setMessageType(analyzeRequest.getMessageType());
+            fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
+
+        } catch (Exception e) {
+            log.warn("Failed to process analyze request: {}", analyzeRequest, e);
+            fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
        }
-
-        result.setMessageType(analyzeRequest.getMessageType());
-        fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
    }


@@ -1,22 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

-import static java.util.stream.Collectors.toSet;
-
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import lombok.Data;
-import lombok.Getter;
-
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

+import lombok.Data;
+import lombok.Getter;

@Data
 public class Dictionary {

-    public static final String RECOMMENDATION_PREFIX = "recommendation_";
-
    @Getter
    private List<DictionaryModel> dictionaryModels;
    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
@@ -26,6 +20,7 @@ public class Dictionary {


    public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) {
+
        this.dictionaryModels = dictionaryModels;
        this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
        this.version = version;
@@ -33,6 +28,7 @@ public class Dictionary {


    public int getDictionaryRank(String type) {
+
        if (!localAccessMap.containsKey(type)) {
            return 0;
        }
@@ -40,16 +36,6 @@ public class Dictionary {
    }


-    public boolean isRecommendation(String type) {
-
-        DictionaryModel model = localAccessMap.get(type);
-        if (model != null) {
-            return model.isRecommendation();
-        }
-        return false;
-    }
-
-
    public boolean hasLocalEntries() {

        return dictionaryModels.stream().anyMatch(dm -> !dm.getLocalEntries().isEmpty());
@@ -62,17 +48,9 @@ public class Dictionary {
    }


-    public boolean containsValue(String type, String value) {
+    public DictionaryModel getType(String type) {

-        return localAccessMap.containsKey(type) && localAccessMap.get(type)
-                .getValues(false)
-                .contains(value) || localAccessMap.containsKey(type) && localAccessMap.get(type)
-                .getValues(true)
-                .contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
-                .getValues(false)
-                .contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
-                .getValues(true)
-                .contains(value);
+        return localAccessMap.get(type);
    }


@@ -85,6 +63,7 @@ public class Dictionary {
        return false;
    }

+
    public boolean isCaseInsensitiveDictionary(String type) {

        DictionaryModel dictionaryModel = localAccessMap.get(type);
@@ -0,0 +1,26 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class DictionaryEntries {
+
+    @Builder.Default
+    Set<DictionaryEntry> entries = new HashSet<>();
+    @Builder.Default
+    Set<DictionaryEntry> falsePositives = new HashSet<>();
+    @Builder.Default
+    Set<DictionaryEntry> falseRecommendations = new HashSet<>();
+
+}
@@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;


 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import lombok.AllArgsConstructor;
 import lombok.Data;

 import java.io.Serializable;
+import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;

@@ -14,19 +16,76 @@ import java.util.stream.Collectors;
@AllArgsConstructor
 public class DictionaryModel implements Serializable {

-    private String type;
-    private int rank;
-    private float[] color;
-    private boolean caseInsensitive;
-    private boolean hint;
-    private boolean recommendation;
-    private Set<DictionaryEntry> entries;
-    private Set<String> localEntries;
-    private boolean isDossierDictionary;
+    private final String type;
+    private final int rank;
+    private final float[] color;
+    private final boolean caseInsensitive;
+    private final boolean hint;
+    private final boolean isDossierDictionary;
+    private final Set<DictionaryEntry> entries;
+    private final Set<DictionaryEntry> falsePositives;
+    private final Set<DictionaryEntry> falseRecommendations;
+
+    private transient SearchImplementation entriesSearch;
+    private transient SearchImplementation falsePositiveSearch;
+    private transient SearchImplementation falseRecommendationsSearch;
+    private transient SearchImplementation localSearch;
+
+    private final Set<String> localEntries = new HashSet<>();
+
+    public DictionaryModel(String type,
+                           int rank,
+                           float[] color,
+                           boolean caseInsensitive,
+                           boolean hint,
+                           Set<DictionaryEntry> entries,
+                           Set<DictionaryEntry> falsePositives,
+                           Set<DictionaryEntry> falseRecommendations,
+                           boolean isDossierDictionary) {
+        this.type = type;
+        this.rank = rank;
+        this.color = color;
+        this.caseInsensitive = caseInsensitive;
+        this.hint = hint;
+        this.isDossierDictionary = isDossierDictionary;
+
+        this.entries = entries;
+        this.falsePositives = falsePositives;
+        this.falseRecommendations = falseRecommendations;
+
+        this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);

-    public Set<String> getValues(boolean local) {
-        return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e -> e.getValue()).collect(Collectors
-                .toSet());
    }

+    public SearchImplementation getLocalSearch() {
+        if (this.localSearch == null) {
+            this.localSearch = new SearchImplementation(this.localEntries, caseInsensitive);
+        }
+        return this.localSearch;
+    }
+
+    public SearchImplementation getEntriesSearch() {
+        if (entriesSearch == null) {
+            this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        }
+        return entriesSearch;
+    }
+
+    public SearchImplementation getFalsePositiveSearch() {
+        if (falsePositiveSearch == null) {
+            this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        }
+        return falsePositiveSearch;
+    }
+
+    public SearchImplementation getFalseRecommendationsSearch() {
+        if (falseRecommendationsSearch == null) {
+            this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), caseInsensitive);
+        }
+        return falseRecommendationsSearch;
+    }
+
+
 }
@@ -48,11 +48,13 @@ public class Entity implements ReasonHolder {

    private Set<Entity> references = new HashSet<>();

+    private EntityType entityType;
+

    public Entity(String word, String type, boolean redaction, String redactionReason,
                  List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber,
                  String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start,
-                  Integer end, boolean isDossierDictionaryEntry, Set<Engine> engines, Set<Entity> references) {
+                  Integer end, boolean isDossierDictionaryEntry, Set<Engine> engines, Set<Entity> references, EntityType entityType) {

        this.word = word;
        this.type = type;
@@ -71,11 +73,12 @@ public class Entity implements ReasonHolder {
        this.isDossierDictionaryEntry = isDossierDictionaryEntry;
        this.engines = engines;
        this.references = references;
+        this.entityType = entityType;
    }


    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber,
-                  boolean isDictionaryEntry, boolean isDossierDictionaryEntry, Engine engine) {
+                  boolean isDictionaryEntry, boolean isDossierDictionaryEntry, Engine engine, EntityType entityType) {

        this.word = word;
        this.type = type;
@@ -86,6 +89,7 @@ public class Entity implements ReasonHolder {
        this.isDictionaryEntry = isDictionaryEntry;
        this.isDossierDictionaryEntry = isDossierDictionaryEntry;
        this.engines.add(engine);
+        this.entityType = entityType;
    }


@@ -0,0 +1,5 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+public enum EntityType {
+    ENTITY, RECOMMENDATION, FALSE_POSITIVE, FALSE_RECOMMENDATION
+}
@@ -1,7 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;

+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;

 import java.util.ArrayList;
@@ -11,6 +13,9 @@ import java.util.regex.Pattern;

 public class SearchableText {

+
+    @JsonIgnore
+    private transient String stringRepresentation;
    private final List<TextPositionSequence> sequences = new ArrayList<>();


@@ -88,8 +93,8 @@ public class SearchableText {
                        .get(i)
                        .charAt(j, caseInsensitive) == '-') {

-                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
-                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(searchSpace.get(i - 1)
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && SeparatorUtils.isSeparator(searchSpace.get(i)
+                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && SeparatorUtils.isSeparator(searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1)
                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
                            .charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
@@ -105,9 +110,9 @@ public class SearchableText {
                        crossSequenceParts.add(partMatch);

                        if (i == searchSpace.size() - 1 && j == searchSpace.get(i)
-                                .length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i)
+                                .length() - 1 || j != searchSpace.get(i).length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i)
                                .charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
-                                .length() - 1 && isSeparator(searchSpace.get(i + 1)
+                                .length() - 1 && SeparatorUtils.isSeparator(searchSpace.get(i + 1)
                                .charAt(0, caseInsensitive)) || j == searchSpace.get(i)
                                .length() - 1 && searchSpace.get(i)
                                .charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
@@ -175,18 +180,18 @@ public class SearchableText {
    }


-    private boolean isSeparator(char c) {
-
-        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
-    }
-
-
    @Override
    public String toString() {
-
        return buildString(sequences);
    }

+    public String asString() {
+        if (stringRepresentation == null) {
+            stringRepresentation = buildString(sequences);
+        }
+        return stringRepresentation;
+    }
+

    public String buildString(List<TextPositionSequence> sequences) {

@@ -3,6 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 import java.util.ArrayList;
 import java.util.List;

+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 import lombok.Data;

@Data
@@ -10,6 +13,9 @@ public class ImageServiceResponse {

    private String dossierId;
    private String fileId;
-    private List<ImageMetadata> imageMetadata = new ArrayList<>();
+
+    @JsonProperty(value = "imageMetadata")
+    @JsonAlias("data")
+    private List<ImageMetadata> data = new ArrayList<>();

 }
@@ -1,17 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@@ -27,6 +32,8 @@ import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
+
@Slf4j
@Service
@RequiredArgsConstructor
@@ -43,11 +50,11 @@ public class AnalyzeService {
    private final RedactionServiceSettings redactionServiceSettings;
    private final SectionTextBuilderService sectionTextBuilderService;
    private final SectionGridCreatorService sectionGridCreatorService;
-    private final NerAnalyserService nerAnalyserService;
    private final ImageService imageService;
+    private final ImportedRedactionService importedRedactionService;


-    public void analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
+    public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {

        long startTime = System.currentTimeMillis();

@@ -55,9 +62,7 @@ public class AnalyzeService {
        Document classifiedDoc;

        try {
-
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
-                    .getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
+            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));

            Map<Integer, List<PdfImage>> pdfImages = null;
            if (redactionServiceSettings.isEnableImageClassification()) {
@@ -75,50 +80,24 @@ public class AnalyzeService {
        Text text = new Text(pageCount, sectionTexts);

        // enhance section grid with headline data
-        sectionTexts.forEach(sectionText -> classifiedDoc
-                .getSectionGrid().getSections().add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
-                        sectionText.getHeadline(),
-                        sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
-                        sectionText.getSectionAreas())));
+        sectionTexts.forEach(sectionText -> classifiedDoc.getSectionGrid()
+                .getSections()
+                .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
+                        .stream()
+                        .map(SectionArea::getPage)
+                        .collect(Collectors.toSet()), sectionText.getSectionAreas())));

+        log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
-                .getSectionGrid());
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());

-        log.info("Document structure analysis successful, took: {}", System.currentTimeMillis() - startTime);
-    }
-
-
-    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
-
-        long startTime = System.currentTimeMillis();
-
-        var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
-            nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        }
-
-        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
-        long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
-                .getDossierId());
-
-        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
-
-        dictionaryService.updateExternalDictionary(dictionary, analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
-                .getDossierTemplateId());
-
-        var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(),
-                redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
-                rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-
-        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
+        return AnalyzeResult.builder()
+                .dossierId(analyzeRequest.getDossierId())
+                .fileId(analyzeRequest.getFileId())
+                .duration(System.currentTimeMillis() - startTime)
+                .numberOfPages(text.getNumberOfPages())
+                .analysisVersion(redactionServiceSettings.getAnalysisVersion())
+                .build();
    }


@@ -135,20 +114,24 @@ public class AnalyzeService {
            return analyze(analyzeRequest);
        }

-        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
-                .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
+//        var dis = System.currentTimeMillis();
+        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
+//        log.info("Dictionary Increment time time: {} ms", (System.currentTimeMillis() - dis));

+//        var fis = System.currentTimeMillis();
        Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse()
                .isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest);
+//        log.info("Find sections time: {} ms", (System.currentTimeMillis() - fis));

        if (sectionsToReanalyse.isEmpty()) {
            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
        }

-        var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
-            nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        } else {
+            nerEntities = NerEntities.builder().build();
        }

        List<SectionText> reanalysisSections = text.getSectionTexts()
@@ -156,21 +139,67 @@ public class AnalyzeService {
                .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
                .collect(Collectors.toList());

+//        long kis = System.currentTimeMillis();
        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+//        log.info("Kie time: {} ms", (System.currentTimeMillis() - kis));

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest
-                .getDossierId());
+//        long dds = System.currentTimeMillis();
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+//        log.info("Dict Time time: {} ms", (System.currentTimeMillis() - dds));

+//        long pis = System.currentTimeMillis();
        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
-        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest
-                .getDossierTemplateId());
+//        log.info("Find Entities time: {}", (System.currentTimeMillis() - pis));

-        redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
-        redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
+//        long crs = System.currentTimeMillis();
+        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
+//        log.info("Create Redaction-log time: {} ms", (System.currentTimeMillis() - crs));
+//        long prs = System.currentTimeMillis();
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newRedactionLogEntries, false);
+//        log.info("Process imports time: {} ms", (System.currentTimeMillis() - prs));
+        redactionLog.getRedactionLogEntry()
+                .removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType()
+                        .equals(IMPORTED_REDACTION_TYPE));
+        redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
+
+//        var fls = System.currentTimeMillis();
+        //        log.info("Finalize time: {} ms", (System.currentTimeMillis() - fls));
        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
    }


+    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+        var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        } else {
+            nerEntities = NerEntities.builder().build();
+        }
+        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+        long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
+        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
+
+        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
+
+        var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, legalBasis, dictionary.getVersion()
+                .getDossierTemplateVersion(), dictionary.getVersion()
+                .getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
+
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog.getRedactionLogEntry(), true);
+        redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
+
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
+    }
+
+
    private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog,
                                                 Text text, AnalyzeRequest analyzeRequest) {

@@ -180,7 +209,7 @@ public class AnalyzeService {
        Set<Integer> sectionsToReanalyse = new HashSet<>();
        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-            if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
+            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
                sectionsToReanalyse.add(entry.getSectionNumber());
            }
            if (entry.isImage()) {
@@ -188,13 +217,19 @@ public class AnalyzeService {
            }
        }

+//        long ss = System.currentTimeMillis();
+
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream()
+                .map(DictionaryIncrementValue::getValue).collect(Collectors.toList()), true);
+
        for (SectionText sectionText : text.getSectionTexts()) {

-            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrement.getValues())) {
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
                sectionsToReanalyse.add(sectionText.getSectionNumber());
            }

        }
+//        log.info("Section Find time: {}", (System.currentTimeMillis() - ss));

        log.info("Should reanalyze {} sections for request: {}, took: {}", sectionsToReanalyse.size(), analyzeRequest, System.currentTimeMillis() - start);

@@ -211,10 +246,8 @@ public class AnalyzeService {

        excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());

-        var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(),
-                redactionLog, analyzeRequest.getAnalysisNumber());
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange
-                .getRedactionLog());
+        var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog, analyzeRequest.getAnalysisNumber());
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());

        long duration = System.currentTimeMillis() - startTime;

@@ -231,6 +264,7 @@ public class AnalyzeService {
                .legalBasisVersion(redactionLog.getLegalBasisVersion())
                .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
                .wasReanalyzed(isReanalysis)
+                .manualRedactions(analyzeRequest.getManualRedactions())
                .build();
    }

@@ -242,13 +276,14 @@ public class AnalyzeService {
        }

        return Stream.concat(manualRedactions.getLegalBasisChanges()
-                        .stream()
-                        .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
-                        .stream()
-                        .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
-                        .stream()
-                        .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
-                .collect(Collectors.toSet());
+                .stream()
+                .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
+                .stream()
+                .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
+                .stream()
+                .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions()
+                .stream()
+                .map(ManualForceRedaction::getAnnotationId)))).collect(Collectors.toSet());
    }


@@ -5,8 +5,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -14,7 +12,8 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;

-import java.awt.Color;
+import java.awt.*;
+import java.util.List;
 import java.util.*;
 import java.util.stream.Collectors;

@@ -36,13 +35,13 @@ public class DictionaryService {
        long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
        var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
        if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
-            updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, null);
+            updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
        }

        long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
        var dossierDictionary = dictionariesByDossier.get(dossierId);
        if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
-            updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, dossierId);
+            updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
        }

        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
@@ -61,9 +60,19 @@ public class DictionaryService {
                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
                }
            });
+            dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> {
+                if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
+                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                }
+            });
+            dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> {
+                if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
+                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                }
+            });
        });

-        if(dictionariesByDossier.containsKey(dossierId)) {
+        if (dictionariesByDossier.containsKey(dossierId)) {
            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
            dictionaryModels.forEach(dictionaryModel -> {
                dictionaryModel.getEntries().forEach(dictionaryEntry -> {
@@ -71,6 +80,16 @@ public class DictionaryService {
                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
                    }
                });
+                dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> {
+                    if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
+                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                    }
+                });
+                dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> {
+                    if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
+                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                    }
+                });
            });
        }

@@ -78,18 +97,61 @@ public class DictionaryService {
    }


-    private void updateDictionaryEntry(String dossierTemplateId, long version, String dossierId) {
+    private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {

        try {
            DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();

-            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId) : dictionaryClient.getAllTypesForDossier(dossierId);
+            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, false)
+                    : dictionaryClient.getAllTypesForDossier(dossierId, false);
            if (CollectionUtils.isNotEmpty(typeResponse)) {

                List<DictionaryModel> dictionary = typeResponse
                        .stream()
-                        .map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
-                                .isHint(), t.isRecommendation(), convertEntries(t.getId()), new HashSet<>(), dossierId != null))
+                        .map(t -> {
+
+                            Optional<DictionaryModel> oldModel;
+                            if (dossierId == null) {
+                                var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
+                                oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
+                            } else {
+                                var representation = dictionariesByDossier.get(dossierId);
+                                oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
+                            }
+
+                            Set<DictionaryEntry> entries = new HashSet<>();
+                            Set<DictionaryEntry> falsePositives = new HashSet<>();
+                            Set<DictionaryEntry> falseRecommendations = new HashSet<>();
+
+                            DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
+
+                            var newValues = newEntries.getEntries().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
+                            var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
+                            var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
+
+
+                            oldModel.ifPresent(oldDictionaryModel -> {
+
+                            });
+                            // add old entries from existing DictionaryModel
+                            oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(
+                                    f -> !newValues.contains(f.getValue())).collect(Collectors.toList())
+                            ));
+                            oldModel.ifPresent(dictionaryModel -> falsePositives.addAll(dictionaryModel.getFalsePositives().stream().filter(
+                                    f -> !newFalsePositivesValues.contains(f.getValue())).collect(Collectors.toList())
+                            ));
+                            oldModel.ifPresent(dictionaryModel -> falseRecommendations.addAll(dictionaryModel.getFalseRecommendations().stream().filter(
+                                    f -> !newFalseRecommendationsValues.contains(f.getValue())).collect(Collectors.toList())
+                            ));
+
+                            // Add Increments
+                            entries.addAll(newEntries.getEntries());
+                            falsePositives.addAll(newEntries.getFalsePositives());
+                            falseRecommendations.addAll(newEntries.getFalseRecommendations());
+
+                            return new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
+                                    .isHint(), entries, falsePositives, falseRecommendations, dossierId != null);
+                        })
                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                        .collect(Collectors.toList());

@@ -105,7 +167,7 @@ public class DictionaryService {
                dictionaryRepresentation.setDictionaryVersion(version);
                dictionaryRepresentation.setDictionary(dictionary);

-                if(dossierId == null) {
+                if (dossierId == null) {
                    dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
                } else {
                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
@@ -118,34 +180,24 @@ public class DictionaryService {
    }


-    public void updateExternalDictionary(Dictionary dictionary, String dossierTemplateId, String dossierId) {
+    private DictionaryEntries getEntries(String typeId, Long fromVersion) {

-        dictionary.getDictionaryModels().forEach(dm -> {
-            if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
-                dictionaryClient.addEntries(IdBuilder.getTypeId(dm.getType(), dossierTemplateId, dm.isDossierDictionary() ? dossierId : null), new ArrayList<>(dm.getLocalEntries()), false, true);
-                long externalVersion = dictionaryClient.getVersion(dossierTemplateId);
-                if (externalVersion == dictionary.getVersion().getDossierTemplateVersion() + 1) {
-                    dictionary.getVersion().setDossierTemplateVersion(externalVersion);
-                }
-            }
-        });
-    }
+        var type = dictionaryClient.getDictionaryForType(typeId, fromVersion);

-
-    private Set<DictionaryEntry> convertEntries(String typeId) {
-
-        var type = dictionaryClient.getDictionaryForType(typeId);
-
-        Set<DictionaryEntry> entries = new HashSet<>(type
-                .getEntries());
+        Set<DictionaryEntry> entries = type.getEntries() != null ? new HashSet<>(type.getEntries()) : new HashSet<>();
+        Set<DictionaryEntry> falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries()) : new HashSet<>();
+        Set<DictionaryEntry> falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries()) : new HashSet<>();

        if (type.isCaseInsensitive()) {
            entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
+            falsePositives.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
+            falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
        }
-        return entries;
+        return new DictionaryEntries(entries, falsePositives, falseRecommendations);
    }


+
    private float[] convertColor(String hex) {

        Color color = Color.decode(hex);
@@ -153,16 +205,6 @@ public class DictionaryService {
    }


-    public boolean isCaseInsensitiveDictionary(String type, String dossierTemplateId) {
-
-        DictionaryModel dictionaryModel = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
-        if (dictionaryModel != null) {
-            return dictionaryModel.isCaseInsensitive();
-        }
-        return false;
-    }
-
-
    public float[] getColor(String type, String dossierTemplateId) {

        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
@@ -183,16 +225,6 @@ public class DictionaryService {
    }


-    public boolean isRecommendation(String type, String dossierTemplateId) {
-
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
-        if (model != null) {
-            return model.isRecommendation();
-        }
-        return false;
-    }
-
-
    public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {

        List<DictionaryModel> copy = new ArrayList<>();
@@ -204,7 +236,7 @@ public class DictionaryService {

        //TODO merge dictionaries if they have same names
        long dossierDictionaryVersion = -1;
-        if(dictionariesByDossier.containsKey(dossierId)) {
+        if (dictionariesByDossier.containsKey(dossierId)) {
            var dossierRepresentation = dictionariesByDossier.get(dossierId);
            dossierRepresentation.getDictionary().forEach(dm -> {
                copy.add(SerializationUtils.clone(dm));
@@ -216,21 +248,17 @@ public class DictionaryService {
    }


-    public float[] getRequestRemoveColor(String dossierTemplateId) {
-
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestRemoveColor();
-    }
-
-
    public float[] getNotRedactedColor(String dossierTemplateId) {

        return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
    }


-    public float[] getRequestAddColor(String dossierTemplateId) {
-
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
+    private Long getVersion(DictionaryRepresentation dictionaryRepresentation) {
+        if (dictionaryRepresentation == null) {
+            return null;
+        } else {
+            return dictionaryRepresentation.getDictionaryVersion();
+        }
    }
-
 }
@@ -1,43 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;

@Slf4j
@Service
@@ -62,7 +46,9 @@ public class EntityRedactionService {
            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
        }

-        Map<Integer, List<Entity>> entitiesPerPage = convertToEnititesPerPage(entities);
+        Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(entities);
+        EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
+
        return new PageEntities(entitiesPerPage, imagesPerPage);
    }

@@ -85,6 +71,7 @@ public class EntityRedactionService {
                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
            }

+
            if (!local && analyzeRequest.getManualRedactions() != null) {

                var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream()
@@ -123,6 +110,7 @@ public class EntityRedactionService {
                }));
            }

+
            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                    .isLocal(false)
                    .dictionaryTypes(dictionary.getTypes())
@@ -141,6 +129,7 @@ public class EntityRedactionService {
                    .images(reanalysisSection.getImages())
                    .fileAttributes(analyzeRequest.getFileAttributes())
                    .build(), reanalysisSection.getSearchableText()));
+
        }

        Set<Entity> entities = new HashSet<>();
@@ -161,7 +150,7 @@ public class EntityRedactionService {
    }


-    private Map<Integer, List<Entity>> convertToEnititesPerPage(Set<Entity> entities) {
+    private Map<Integer, List<Entity>> convertToEntitiesPerPage(Set<Entity> entities) {

        Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
        for (Entity entity : entities) {
@@ -177,7 +166,7 @@ public class EntityRedactionService {
                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
                                .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity
-                                .getReferences()));
+                                .getReferences(), entity.getEntityType()));
            }
        }
        return entitiesPerPage;
@@ -199,90 +188,76 @@ public class EntityRedactionService {
    private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {

        analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> {
-            if (dictionary.isRecommendation(key)) {
-                analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
-                    if (!dictionary.containsValue(key, value)) {
-                        dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
-                    }
-                });
-            } else {
-                analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
+            analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {

-                    if (dictionary.getLocalAccessMap().get(key) == null) {
-                        log.warn("Dictionary {} is null", key);
-                    }
+                if (dictionary.getLocalAccessMap().get(key) == null) {
+                    log.warn("Dictionary {} is null", key);
+                }

-                    if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
-                        log.warn("Dictionary {} localEntries is null", key);
-                    }
+                if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
+                    log.warn("Dictionary {} localEntries is null", key);
+                }

-                    dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
-                });
-            }
+                dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
+            });
        });
    }


    private Entities findEntities(SearchableText searchableText, String headline, int sectionNumber,
                                  Dictionary dictionary, boolean local, NerEntities nerEntities,
-                                  List<Integer> cellstarts) {
+                                  List<Integer> cellStarts) {

        Set<Entity> found = new HashSet<>();
-        String searchableString = searchableText.toString();
+        String searchableString = searchableText.asString();
+
        if (StringUtils.isEmpty(searchableString)) {
            return new Entities(new HashSet<>(), new HashSet<>());
        }

        String lowercaseInputString = searchableString.toLowerCase();
        for (DictionaryModel model : dictionary.getDictionaryModels()) {
-            if (model.isCaseInsensitive()) {
-                EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model
-                        .getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
-            } else {
-                EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(searchableString, model.getValues(local), model
-                        .getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
-            }
+
+            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation, model, new FindEntityDetails(model.getType(),headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, local? EntityType.RECOMMENDATION: EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
        }

        Set<Entity> nerFound = new HashSet<>();
        if (!local) {
-            Map<String, Set<String>> nerValuesPerType = getNerValues(sectionNumber, nerEntities, cellstarts);
-            nerValuesPerType.entrySet().forEach(entry -> {
-                EntitySearchUtils.addEntitiesWithHigherRank(nerFound, EntitySearchUtils.find(searchableString, entry.getValue(), entry
-                        .getKey(), headline, sectionNumber, false, false, Engine.NER), dictionary);
-            });
+            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
        }

-        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound) ;
+        return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound);
    }


-    private Map<String, Set<String>> getNerValues(int sectionNumber, NerEntities nerEntities,
-                                                  List<Integer> cellstarts) {
+    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts,
+                                     String headline) {

-        Map<String, Set<String>> nerValuesPerType = new HashMap<>();
+        Set<Entity> entities = new HashSet<>();

-        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities.getResult()
-                .containsKey(sectionNumber)) {
-            nerEntities.getResult().get(sectionNumber).forEach(res -> {
-                if (cellstarts == null || cellstarts.isEmpty()) {
-                    nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
-                            .add(new String(Base64.decodeBase64(res.getValue().getBytes())));
+        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
+            nerEntities.getData().get(sectionNumber).forEach(res -> {
+                if (cellStarts == null || cellStarts.isEmpty()) {
+                    entities.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
                } else {
                    boolean intersectsCellStart = false;
-                    for (Integer cellStart : cellstarts) {
+                    for (Integer cellStart : cellStarts) {
                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
                            intersectsCellStart = true;
+                            break;
                        }
                    }
                    if (!intersectsCellStart) {
-                        nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
-                                .add(new String(Base64.decodeBase64(res.getValue().getBytes())));
+                        entities.add(new Entity(res.getValue(), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER, EntityType.RECOMMENDATION));
                    }
                }
            });
        }
-        return nerValuesPerType;
+        return entities;
    }

 }
@@ -0,0 +1,105 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.util.HashSet;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
+import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class ImportedRedactionService {
+
+    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
+
+    private final DictionaryService dictionaryService;
+    private final RedactionStorageService redactionStorageService;
+
+
+    public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId, String dossierId, String fileId,
+                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                             boolean addImportedRedactions) {
+
+        var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
+        if (importedRedactions == null) {
+            return redactionLogEntries;
+        }
+
+        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
+
+        if (addImportedRedactions) {
+            return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
+        }
+
+        return redactionLogEntries;
+    }
+
+
+    private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
+                                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                                             ImportedRedactions importedRedactions) {
+
+        for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
+            for (ImportedRedaction importedRedaction : importedRedactionsValues) {
+                RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
+                        .id(importedRedaction.getId())
+                        .type(IMPORTED_REDACTION_TYPE)
+                        .imported(true)
+                        .redacted(true)
+                        .positions(importedRedaction.getPositions())
+                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
+                        .build();
+
+                redactionLogEntries.add(redactionLogEntry);
+            }
+        }
+
+        return redactionLogEntries;
+    }
+
+
+    private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
+
+        for (Rectangle rectangle : redactionLogEntry.getPositions()) {
+            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
+                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
+                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
+                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                        if (intersects(importedRedactionPosition, rectangle)) {
+                            if(redactionLogEntry.getImportedRedactionIntersections() == null){
+                                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                            }
+                            redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    private boolean intersects(Rectangle importedPosition, Rectangle redactionLogPosition) {
+
+        return redactionLogPosition.getTopLeft()
+                .getX() + redactionLogPosition.getWidth() > importedPosition.getTopLeft()
+                .getX() && redactionLogPosition.getTopLeft()
+                .getY() + redactionLogPosition.getHeight() > importedPosition.getTopLeft()
+                .getY() && redactionLogPosition.getTopLeft().getX() < importedPosition.getTopLeft()
+                .getX() + importedPosition.getWidth() && redactionLogPosition.getTopLeft()
+                .getY() < importedPosition.getTopLeft().getY() + importedPosition.getHeight();
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId) {
+
+        return dictionaryService.getColor(type, dossierTemplateId);
+    }
+
+}
@@ -1,16 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -18,11 +12,19 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;

@Slf4j
@Service
@@ -33,8 +35,9 @@ public class ManualRedactionSurroundingTextService {
    private final SurroundingWordsService surroundingWordsService;


-    public ManualRedactions addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
+    public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {

+        long startTime = System.currentTimeMillis();
        Text text = redactionStorageService.getText(dossierId, fileId);
        List<ManualRedactionEntry> processedAddRedactions = new ArrayList<>();
        List<ManualResizeRedaction> processedResizeRedactions = new ArrayList<>();
@@ -51,8 +54,7 @@ public class ManualRedactionSurroundingTextService {
                while (addItty.hasNext()) {
                    var manualAddRedaction = addItty.next();
                    if (sectionContainsEntry(sectionArea, manualAddRedaction.getPositions())) {
-                        var surroundingText = findSurroundingText(sectionText, manualAddRedaction.getValue(), manualAddRedaction
-                                .getPositions());
+                        var surroundingText = findSurroundingText(sectionText, manualAddRedaction.getValue(), manualAddRedaction.getPositions());
                        manualAddRedaction.setTextBefore(surroundingText.getLeft());
                        manualAddRedaction.setTextAfter(surroundingText.getRight());
                        processedAddRedactions.add(manualAddRedaction);
@@ -64,8 +66,7 @@ public class ManualRedactionSurroundingTextService {
                while (resizeItty.hasNext()) {
                    var manualResizeRedaction = resizeItty.next();
                    if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
-                        var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction
-                                .getPositions());
+                        var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
                        manualResizeRedaction.setTextBefore(surroundingText.getLeft());
                        manualResizeRedaction.setTextAfter(surroundingText.getRight());
                        processedResizeRedactions.add(manualResizeRedaction);
@@ -77,27 +78,31 @@ public class ManualRedactionSurroundingTextService {

        manualRedactions.getEntriesToAdd().addAll(processedAddRedactions);
        manualRedactions.getResizeRedactions().addAll(processedResizeRedactions);
-        return manualRedactions;
+
+        return AnalyzeResult.builder()
+                .dossierId(dossierId)
+                .fileId(fileId)
+                .manualRedactions(manualRedactions)
+                .duration(System.currentTimeMillis() - startTime)
+                .build();
    }


-    private Pair<String, String> findSurroundingText(SectionText sectionText, String value,
-                                                     List<Rectangle> toFindPositions) {
+    private Pair<String, String> findSurroundingText(SectionText sectionText, String value, List<Rectangle> toFindPositions) {

-        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), Set.of(value), "dummy", sectionText.getHeadline(), sectionText
-                .getSectionNumber(), false, false, Engine.DICTIONARY);
+        Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), new SearchImplementation(value, false),
+                new FindEntityDetails("dummy", sectionText.getHeadline(), sectionText.getSectionNumber(), false, false, Engine.DICTIONARY, EntityType.ENTITY));
        Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null);

        Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);

-        if(correctEntity == null){
+        if (correctEntity == null) {
            log.warn("Could not calculate surrounding text");
-            return Pair.of("","");
+            return Pair.of("", "");
        }

        if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText
-                    .getCellStarts());
+            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
        } else {
            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
        }
@@ -1,53 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
-import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
-import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionSection;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.codec.binary.Base64;
-import org.springframework.stereotype.Service;
-
-import java.util.stream.Collectors;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class NerAnalyserService {
-
-    private final RedactionStorageService redactionStorageService;
-    private final EntityRecognitionClient entityRecognitionClient;
-    private final RedactionServiceSettings redactionServiceSettings;
-
-    public void computeNerEntities(String dossierId, String fileId) {
-
-        if (redactionServiceSettings.isEnableEntityRecognition()) {
-            var text = redactionStorageService.getText(dossierId, fileId);
-
-            long start = System.currentTimeMillis();
-
-            if (text != null) {
-                var nerRequest = EntityRecognitionRequest.builder()
-                        .data(text.getSectionTexts()
-                                .stream()
-                                .map(sectionText -> new EntityRecognitionSection(sectionText.getSectionNumber(), new String(Base64
-                                        .encodeBase64(sectionText
-                                                .getText().getBytes()))))
-                                .collect(Collectors.toList()))
-                        .build();
-
-                var nerResponse = entityRecognitionClient.findAuthors(nerRequest);
-
-                log.info("Computing NER entities took: {} ms for dossierId {} and fileId {}", System.currentTimeMillis() - start, dossierId, fileId);
-
-                redactionStorageService.storeObject(dossierId, fileId, FileType.NER_ENTITIES, nerResponse);
-            } else {
-                log.warn("Warning, text for file: {} in dossier: {} is null", fileId, dossierId);
-            }
-        }
-    }
-
-}
@@ -33,17 +33,11 @@ public class RedactionChangeLogService {

        List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
                .stream()
-                .filter(entry -> !entry.getChanges()
-                        .get(entry.getChanges().size() - 1)
-                        .getType()
-                        .equals(ChangeType.REMOVED))
+                .filter(entry -> !entry.lastChangeIsRemoved())
                .collect(Collectors.toList());

        Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream()
-                .filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges()
-                        .get(entry.getChanges().size() - 1)
-                        .getType()
-                        .equals(ChangeType.REMOVED)).collect(Collectors.toSet());
+                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()).collect(Collectors.toSet());

        notRemovedPreviousEntries.forEach(added::remove);

@@ -87,7 +81,7 @@ public class RedactionChangeLogService {

        addedIds.forEach((k, v) -> {
            if (v.getChanges().isEmpty()) {
-                v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now()));
+                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
            }
            newRedactionLogEntries.add(v);
        });
@@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
@@ -70,7 +71,6 @@ public class RedactionLogCreatorService {
                    .legalBasis(image.getLegalBasis())
                    .matchedRule(image.getMatchedRule())
                    .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
-                    .manual(false)
                    .isDictionaryEntry(false)
                    .isRecommendation(false)
                    .positions(List.of(new Rectangle(new Point((float) image.getPosition()
@@ -92,7 +92,7 @@ public class RedactionLogCreatorService {

        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();

-        // Duplicates can exist due table extraction colums over multiple rows.
+        // Duplicates can exist due table extraction columns over multiple rows.
        Set<String> processedIds = new HashSet<>();

        entityLoop:
@@ -173,10 +173,7 @@ public class RedactionLogCreatorService {
    private boolean isCharClose(float x, float xDirAdj, float width) {

        float max = x + (5 * width);
-        if (xDirAdj < max) {
-            return true;
-        }
-        return false;
+        return xDirAdj < max;
    }


@@ -190,11 +187,7 @@ public class RedactionLogCreatorService {
        float minY = y - offsetY;
        float maxY = y + offsetY;

-        if (yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight) {
-            return true;
-        }
-
-        return false;
+        return yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight;
    }


@@ -211,7 +204,8 @@ public class RedactionLogCreatorService {
                .type(entity.getType())
                .redacted(entity.isRedaction())
                .isHint(isHint(entity.getType(), dossierTemplateId))
-                .isRecommendation(isRecommendation(entity.getType(), dossierTemplateId))
+                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
+                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
                .section(entity.getHeadline())
                .sectionNumber(entity.getSectionNumber())
                .matchedRule(entity.getMatchedRule())
@@ -241,10 +235,4 @@ public class RedactionLogCreatorService {
        return dictionaryService.isHint(type, dossierTemplateId);
    }

-
-    private boolean isRecommendation(String type, String dossierTemplateId) {
-
-        return dictionaryService.isRecommendation(type, dossierTemplateId);
-    }
-
 }
@@ -1,15 +1,26 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;

+import java.awt.*;
 import java.time.OffsetDateTime;
+import java.util.List;
 import java.util.*;
 import java.util.stream.Collectors;

@@ -18,19 +29,55 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
 public class RedactionLogMergeService {

-    private final DictionaryService dictionaryService;

    private final SectionTextService sectionTextService;

+    private final RedactionStorageService redactionStorageService;

-    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
-                                              String dossierTemplateId, ManualRedactions manualRedactions,
-                                              Set<Integer> excludedPages) {
+    public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
+        log.debug("Requested preview for: {}", redactionRequest);
+
+        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
+
+        if (redactionLog == null) {
+            throw new NotFoundException("RedactionLog not present");
+        }
+
+        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
+        if (sectionGrid.getSections().isEmpty()) {
+
+            log.info("SectionGrid does not have headlines set. Computing headlines now!");
+            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
+
+            // enhance section grid with headline data
+            for (var sectionText : text.getSectionTexts()) {
+                sectionGrid.getSections()
+                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), sectionText.getHeadline(), sectionText.getSectionAreas()
+                                .stream()
+                                .map(SectionArea::getPage)
+                                .collect(Collectors.toSet()), sectionText.getSectionAreas()));
+            }
+            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
+        }
+
+        log.debug("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
+        var merged = mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
+
+        merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
+
+        return merged;
+    }
+
+    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
+                                              Set<Integer> excludedPages, List<Type> types, Colors colors) {
+
+        var skippedImportedRedactions = new HashSet<>();

        log.info("Merging Redaction log with manual redactions");
        if (manualRedactions != null) {

-            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);
+
+            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);

            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);

@@ -40,13 +87,17 @@ public class RedactionLogMergeService {

                processRedactionLogEntry(manualRedactionWrappers.stream()
                        .filter(mr -> entry.getId().equals(mr.getId()))
-                        .collect(Collectors.toList()), dossierTemplateId, entry);
+                        .collect(Collectors.toList()), entry, types, colors);
+
+                if (entry.isImported() && !entry.isRedacted()) {
+                    skippedImportedRedactions.add(entry.getId());
+                }

                entry.setComments(manualRedactions.getComments().get(entry.getId()));

                if (excludedPages != null && !excludedPages.isEmpty()) {
                    entry.getPositions().forEach(pos -> {
-                        if (!entry.isManual() && excludedPages.contains(pos.getPage())) {
+                        if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
                            entry.setExcluded(true);
                        }
                    });
@@ -57,6 +108,12 @@ public class RedactionLogMergeService {

        Set<String> processedIds = new HashSet<>();
        redactionLog.getRedactionLogEntry().removeIf(entry -> {
+            if (entry.getImportedRedactionIntersections() != null) {
+                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
+                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) {
+                    return true;
+                }
+            }
            if (processedIds.contains(entry.getId())) {
                log.info("Duplicate annotation found with id {}", entry.getId());
                return true;
@@ -109,7 +166,7 @@ public class RedactionLogMergeService {


    private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers,
-                                          String dossierTemplateId, RedactionLogEntry redactionLogEntry) {
+                                          RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {

        manualRedactionWrappers.forEach(mrw -> {

@@ -117,12 +174,12 @@ public class RedactionLogMergeService {
                var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
                String manualOverrideReason = null;
                if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
+
+
                    redactionLogEntry.setType(imageRecategorization.getType());
-                    redactionLogEntry.setHasBeenRecategorized(true);
                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());

-                    if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
+                    if (isHint(types, imageRecategorization.getType())) {
                        redactionLogEntry.setRedacted(false);
                        redactionLogEntry.setHint(true);
                    } else {
@@ -132,77 +189,65 @@ public class RedactionLogMergeService {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
                } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
-                    redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
                }

-                redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE);
+
+                redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
+                        .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
+                        .withChange("type", imageRecategorization.getType()));
            }

            if (mrw.getItem() instanceof IdRemoval) {
                var manualRemoval = (IdRemoval) mrw.getItem();
                if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
-                    log.info("Skipping merge for dictionary-modifying entry");
+                    log.debug("Skipping merge for dictionary-modifying entry");
                } else {
                    String manualOverrideReason = null;
                    if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
                        redactionLogEntry.setRedacted(false);
-                        redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
+                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), true, types));
                        redactionLogEntry.setHint(false);
-                        redactionLogEntry.setHasBeenRemovedByManualOverride(true);
                    } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                        redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
-                    } else {
-                        redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
+                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                    }

                    redactionLogEntry.setReason(manualOverrideReason);
-                    redactionLogEntry.setManual(true);
-                    redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser());
-                    redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
-                    redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary());
-                    redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
+
+                    redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
+                            .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
+                                    ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
                }
            }

            if (mrw.getItem() instanceof ManualForceRedaction) {
                var manualForceRedact = (ManualForceRedaction) mrw.getItem();
                String manualOverrideReason = null;
+                var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
                if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
                    // Forcing a skipped hint should result in a hint
-                    if (dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId)) {
+                    if (dictionaryIsHint) {
                        redactionLogEntry.setHint(true);
                    } else {
                        redactionLogEntry.setRedacted(true);
                    }
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-                    redactionLogEntry.setHasBeenForced(true);
                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
+                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
                }

-                redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
+                var manualChange = ManualChange.from(manualForceRedact)
+                        .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
+
+                redactionLogEntry.getManualChanges().add(manualChange);

            }

@@ -210,11 +255,9 @@ public class RedactionLogMergeService {
                var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
                String manualOverrideReason = null;
                if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
                    redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
                    redactionLogEntry.setRedacted(true);
-                    redactionLogEntry.setHasLegalBasisChanged(true);
                    if (manualLegalBasisChange.getSection() != null) {
                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
                    }
@@ -223,47 +266,45 @@ public class RedactionLogMergeService {
                    }
                } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
-                    redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                }

-                redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
+
+                var manualChange = ManualChange.from(manualLegalBasisChange)
+                        .withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
+                manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
+                if (manualLegalBasisChange.getSection() != null) {
+                    manualChange.withChange("section", manualLegalBasisChange.getSection());
+                }
+                if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
+                    manualChange.withChange("value", manualLegalBasisChange.getValue());
+                }
+                redactionLogEntry.getManualChanges().add(manualChange);
            }

            if (mrw.getItem() instanceof ManualResizeRedaction) {
                var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
                String manualOverrideReason = null;
                if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                    redactionLogEntry.setValue(manualResizeRedact.getValue());
-                    redactionLogEntry.setHasBeenResized(true);
                    redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
                    redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
                } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-                    redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                    redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
                    redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                } else {
-                    redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
                }

-                redactionLogEntry.setManualRedactionUserId(manualResizeRedact.getUser());
                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.setManual(true);
-                redactionLogEntry.setManualRedactionType(ManualRedactionType.RESIZE);
-
+                redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
+                        .withManualRedactionType(ManualRedactionType.RESIZE)
+                        .withChange("value", manualResizeRedact.getValue()));
            }

        });
@@ -285,14 +326,14 @@ public class RedactionLogMergeService {


    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
-                                                       Map<String, List<Comment>> comments, String dossierTemplateId) {
+                                                       Map<String, List<Comment>> comments, Colors colors, List<Type> types) {

        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();

        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {

-            if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
+            if (shouldCreateManualEntry(manualRedactionEntry)) {
+                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types);
                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
                redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
@@ -317,65 +358,103 @@ public class RedactionLogMergeService {
    }


-    private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
-
-        return manualRedactionEntry.getStatus()
-                .equals(AnnotationStatus.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
+    @SuppressWarnings("PMD.UselessParentheses")
+    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
+        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
+                || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null);
    }


    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
-                                                      String dossierTemplateId) {
+                                                      Colors colors, List<Type> types) {
+
+        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
+
+        var change = ManualChange.from(manualRedactionEntry)
+                .withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
+        List<ManualChange> changeList = new ArrayList<>();
+        changeList.add(change);

        return RedactionLogEntry.builder()
                .id(id)
-                .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
+                .color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
                .reason(manualRedactionEntry.getReason())
                .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
                .legalBasis(manualRedactionEntry.getLegalBasis())
                .value(manualRedactionEntry.getValue())
+                .sourceId(manualRedactionEntry.getSourceId())
                .section(manualRedactionEntry.getSection())
                .type(manualRedactionEntry.getType())
                .redacted(true)
                .isHint(false)
                .sectionNumber(-1)
-                .manual(true)
-                .status(manualRedactionEntry.getStatus())
-                .manualRedactionType(ManualRedactionType.ADD)
-                .manualRedactionUserId(manualRedactionEntry.getUser())
                .rectangle(manualRedactionEntry.isRectangle())
+                .manualChanges(changeList)
                .build();
    }


-    private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction,
-                             boolean skipped) {
+    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction,
+                             boolean skipped, List<Type> types) {

        if (requested) {
-            return dictionaryService.getRequestRemoveColor(dossierTemplateId);
+            return convertColor(colors.getRequestRemove());
        }
-        if (skipped || !isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
+        if (skipped || !isRedaction && !isHint(types, type)) {
+            return convertColor(colors.getNotRedacted());
        }
-        return dictionaryService.getColor(type, dossierTemplateId);
+        return getColor(types, type);
    }


-    private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) {
+    private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {

        if (status.equals(AnnotationStatus.REQUESTED)) {
-            return dictionaryService.getRequestAddColor(dossierTemplateId);
+            return convertColor(colors.getRequestAdd());
        } else if (status.equals(AnnotationStatus.DECLINED)) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
+            return convertColor(colors.getNotRedacted());
        }
-        return getColor(type, dossierTemplateId);
+        return getColor(types, type);
    }


-    private float[] getColor(String type, String dossierTemplateId) {
+    private float[] getColor(List<Type> types, String type) {

-        return dictionaryService.getColor(type, dossierTemplateId);
+        var matchingTypes = getMatchingTypes(types, type);
+        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
+        if (foundAndNotDeletedType.isPresent()) {
+            return convertColor(foundAndNotDeletedType.get().getHexColor());
+        }
+        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
+        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0"));
+    }
+
+
+    boolean isHint(List<Type> types, String type) {
+
+        var matchingTypes = getMatchingTypes(types, type);
+        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
+        if (foundAndNotDeletedType.isPresent()) {
+            return foundAndNotDeletedType.get().isHint();
+        }
+        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
+        return firstDeletedType.map(Type::isHint).orElse(false);
+    }
+
+
+    private float[] convertColor(String hex) {
+
+        Color color = Color.decode(hex);
+        return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
+    }
+
+    private List<Type> getMatchingTypes(List<Type> types, String type) {
+        return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
+    }
+
+    private boolean isDeletedType(Type type) {
+        return type.getSoftDeletedTime() != null;
    }


@@ -1,10 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.List;
-
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.SectionRectangle;
@@ -14,8 +9,11 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;

@Service
@RequiredArgsConstructor
@@ -1,35 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
-import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.classification.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;

@Slf4j
@Service
@@ -191,7 +175,7 @@ public class SectionTextBuilderService {
            sectionText.getSectionAreas().add(sectionArea);
        }

-        sectionText.setText(searchableText.toString());
+        sectionText.setText(searchableText.asString());
        sectionText.setHeadline(headline);
        sectionText.setSectionNumber(sectionNumber.intValue());
        sectionText.setTable(false);
@@ -31,7 +31,7 @@ public class SurroundingWordsService {
                if (dictionary != null && dictionary.isHint(entity.getType())) {
                    continue;
                }
-                findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
+                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
            }
        } catch (Exception e) {
            log.warn("Could not get surrounding text!");
@@ -47,7 +47,7 @@ public class SurroundingWordsService {
        }

        try {
-            String searchableString = searchableText.toString();
+            String searchableString = searchableText.asString();

            if (cellstarts != null) {
                for (int i = 0; i < cellstarts.size(); i++) {
@@ -1,90 +1,57 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;

+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
@Slf4j
@UtilityClass
@SuppressWarnings("PMD")
 public class EntitySearchUtils {

-    public boolean sectionContainsAny(String sectionText, Set<DictionaryIncrementValue> values) {

-        String inputString = sectionText.toLowerCase(Locale.ROOT);
-
-        for (DictionaryIncrementValue value : values) {
-
-            String cleanValue = value.getValue().toLowerCase(Locale.ROOT).trim();
-
-            if (cleanValue.length() <= 2) {
-                continue;
-            }
-
-            int startIndex;
-            int stopIndex = 0;
-            do {
-                startIndex = inputString.indexOf(cleanValue, stopIndex);
-                stopIndex = startIndex + cleanValue.length();
-
-                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    return true;
-                }
-            } while (startIndex > -1);
-        }
-        return false;
+    public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) {
+        return searchImplementation.atLeastOneMatches(sectionText);
    }

+    public Set<Entity> findEntities(String inputString, SearchImplementation searchImplementation, DictionaryModel type, FindEntityDetails details) {

-    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                            boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine) {
+        Set<Entity> found = find(inputString, searchImplementation, details);

-        Set<Entity> found = new HashSet<>();
-
-        for (String value : values) {
-
-            String cleanValue = value.trim();
-
-            if (cleanValue.length() <= 2) {
-                continue;
-            }
-
-            int startIndex;
-            int stopIndex = 0;
-            do {
-                startIndex = inputString.indexOf(cleanValue, stopIndex);
-                stopIndex = startIndex + cleanValue.length();
-
-                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
-                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine));
-                }
-            } while (startIndex > -1);
+        if (details.getEntityType() == EntityType.RECOMMENDATION) {
+            Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
+            removeFalsePositives(found, falseRecommendations);
+            found.addAll(falseRecommendations);
+        } else {
+            Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
+            removeFalsePositives(found, falsePositives);
+            found.addAll(falsePositives);
        }
+
        return found;
    }

+    public Set<Entity> find(String inputString, SearchImplementation searchImplementation, FindEntityDetails findEntityDetails) {
+        Set<Entity> entities = new HashSet<>();

-    private boolean isSeparator(char c) {
+        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex()));

-        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
+        return entities;
+    }
+
+    private void validateAndAddEntity(Set<Entity> entities, FindEntityDetails findEntityDetails, String inputString, int startIndex, int stopIndex) {
+
+        if ((startIndex == 0 || SeparatorUtils.isSeparator(inputString.charAt(startIndex - 1)))
+                && (stopIndex == inputString.length() || SeparatorUtils.isSeparator(inputString.charAt(stopIndex)))) {
+            entities.add(new Entity(inputString.substring(startIndex, stopIndex), findEntityDetails.getType(), startIndex, stopIndex,
+                    findEntityDetails.getHeadline(), findEntityDetails.getSectionNumber(), findEntityDetails.isDictionaryEntry(),
+                    findEntityDetails.isDossierDictionary(), findEntityDetails.getEngine(), findEntityDetails.getEntityType()));
+        }
    }


@@ -98,13 +65,10 @@ public class EntitySearchUtils {

        for (String word : entitiesByWord.keySet()) {

-            List<Entity> orderedEntities = entitiesByWord.get(word)
-                    .stream()
-                    .sorted(Comparator.comparing(Entity::getStart))
-                    .collect(Collectors.toList());
+            List<Entity> orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
            Entity firstEntity = orderedEntities.get(0);
            List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord()
-                    .trim(), dictionary == null ? true : dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
+                    .trim(), dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());

            for (int i = 0; i <= orderedEntities.size() - 1; i++) {
                try {
@@ -121,10 +85,10 @@ public class EntitySearchUtils {
    }


-    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+    public void removeFalsePositives(Set<Entity> entities, Set<Entity> falsePositives) {

        List<Entity> wordsToRemove = new ArrayList<>();
-        for (Entity word : entities) {
+        for (Entity word : falsePositives) {
            for (Entity inner : entities) {
                if (inner.getWord().length() < word.getWord()
                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
@@ -132,7 +96,56 @@ public class EntitySearchUtils {
                }
            }
        }
-        entities.removeAll(wordsToRemove);
+        wordsToRemove.forEach(entities::remove);
+        entities.removeAll(falsePositives);
+    }
+
+
+    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
+        List<Entity> wordsToRemove = new ArrayList<>();
+        for (Entity word : entities) {
+            for (Entity inner : entities) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
+                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
+                        wordsToRemove.add(word);
+                    } else if(!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY ||
+                            inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) {
+                        wordsToRemove.add(inner);
+                    }
+                }
+            }
+        }
+        wordsToRemove.stream().filter(e -> !e.getEntityType().equals(EntityType.FALSE_POSITIVE) && !e.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)).forEach(entities::remove);
+    }
+
+
+    public static void removeEntitiesContainedInRedactedLogos(Map<Integer, Set<Image>> imagesPerPage, Map<Integer, List<Entity>> entitiesPerPage) {
+
+        if (entitiesPerPage != null && imagesPerPage != null) {
+            for (int page : imagesPerPage.keySet()) {
+                Set<Entity> removeEntitiesFromPage = new HashSet<>();
+
+                if (!entitiesPerPage.containsKey(page)) {
+                    continue;
+                }
+                imagesPerPage.get(page).forEach(image -> {
+                    if (image.isRedaction()) {
+                        entitiesPerPage.get(page).forEach(entity -> {
+                            if (isImageOverlappingEntity(image, entity)) {
+                                log.info("Logo overlaps entity and entity will be removed");
+                                removeEntitiesFromPage.add(entity);
+                            }
+
+                        });
+                    }
+                });
+
+                entitiesPerPage.get(page).removeAll(removeEntitiesFromPage);
+
+            }
+        }
    }


@@ -146,13 +159,20 @@ public class EntitySearchUtils {

        if (entities.contains(found)) {
            Optional<Entity> existingOptional = entities.stream().filter(entity -> entity.equals(found)).findFirst();
-            if (!existingOptional.isPresent()) {
+            if (existingOptional.isEmpty()) {
                return;
            }
            var existing = existingOptional.get();

            if (existing.getType().equals(found.getType())) {
                existing.getEngines().addAll(found.getEngines());
+                if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
+                        || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
+                    existing.setEntityType(EntityType.ENTITY);
+                    if (found.isRedaction()) {
+                        existing.setRedaction(true);
+                    }
+                }
            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
                entities.remove(found);
                entities.add(found);
@@ -170,22 +190,17 @@ public class EntitySearchUtils {
    }


-    public void addOrAddEngine(Set<Entity> existing, Set<Entity> toBeAdded) {
+    private boolean isImageOverlappingEntity(Image image, Entity entity) {

-        for (Entity toAdd : toBeAdded) {
-            if (existing.contains(toAdd)) {
-                Optional<Entity> existingOptional = existing.stream()
-                        .filter(entity -> entity.equals(toAdd))
-                        .findFirst();
-                if (!existingOptional.isPresent()) {
-                    return;
-                }
-                var existingEntity = existingOptional.get();
-                existingEntity.getEngines().addAll(toAdd.getEngines());
-            } else {
-                existing.add(toAdd);
-            }
-        }
+        return image.getPosition() != null && entity.getPositionSequences() != null && entity.getPositionSequences().get(0) != null && entity.getPositionSequences()
+                .get(0)
+                .getSequences() != null && entity.getPositionSequences().get(0).getSequences().get(0) != null && image.getPosition().getX() < entity.getPositionSequences()
+                .get(0)
+                .getSequences()
+                .get(0)
+                .getX1() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences().get(0).getSequences().get(0).getX2() && image.getPosition()
+                .getY() < entity.getPositionSequences().get(0).getSequences().get(0).getY1() && image.getPosition().getY() + image.getPosition()
+                .getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getY2();
    }


@@ -205,11 +220,32 @@ public class EntitySearchUtils {
    }


+    public void addOrAddEngine(Set<Entity> existing, Set<Entity> toBeAdded) {
+
+        for (Entity toAdd : toBeAdded) {
+            if (existing.contains(toAdd)) {
+                Optional<Entity> existingOptional = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst();
+                if (existingOptional.isEmpty()) {
+                    return;
+                }
+                var existingEntity = existingOptional.get();
+                existingEntity.getEngines().addAll(toAdd.getEngines());
+            } else {
+                existing.add(toAdd);
+            }
+        }
+    }
+
+    /*
+     * returns true if the found entity overlaps with an existing entity in a way
+     * that neither entity is a subset of the other
+     */
    private boolean overlaps(Set<Entity> existingEntities, Entity found) {

        for (Entity existing : existingEntities) {

-            if(existing.getStart().equals(found.getStart())){
+            // skip if either start or end is equal
+            if (existing.getStart().equals(found.getStart()) || existing.getEnd().equals(found.getEnd())) {
                continue;
            }

@@ -0,0 +1,24 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class FindEntityDetails {
+
+    private String type;
+    private String headline;
+    private int sectionNumber;
+    private boolean isDictionaryEntry;
+    private boolean isDossierDictionary;
+    private Engine engine;
+    private EntityType entityType;
+
+    public FindEntityDetails withEntityType(EntityType entityType) {
+        this.entityType = entityType;
+        return this;
+    }
+}
@@ -0,0 +1,92 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import org.ahocorasick.trie.Trie;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+@Data
+public class SearchImplementation {
+
+    private boolean ignoreCase;
+    private List<String> values;
+
+    private Pattern pattern;
+    private Trie trie;
+
+    public SearchImplementation(String value, boolean ignoreCase) {
+        this.values = List.of(value);
+        this.ignoreCase = ignoreCase;
+        this.createSearchImplementation();
+    }
+
+    public SearchImplementation(Collection<String> values, boolean ignoreCase) {
+        this.values = new ArrayList<>(values);
+        this.ignoreCase = ignoreCase;
+        this.createSearchImplementation();
+    }
+
+    private void createSearchImplementation() {
+        if (this.values.isEmpty()) {
+            return;
+        }
+
+        if (this.values.size() == 1) {
+            var text = this.values.iterator().next();
+            this.pattern = Pattern.compile(Pattern.quote(ignoreCase ? text.toLowerCase(Locale.ROOT) : text));
+        } else {
+            var builder = Trie.builder();
+            if (this.ignoreCase) {
+                builder.ignoreCase();
+            }
+
+            builder.addKeywords(this.values);
+
+            this.trie = builder.build();
+        }
+    }
+
+    public boolean atLeastOneMatches(String text) {
+        String textToCheck = text;
+        if (this.values.isEmpty()) {
+            return false;
+        }
+        if (this.pattern != null) {
+            if (ignoreCase) {
+                textToCheck = textToCheck.toLowerCase(Locale.ROOT);
+            }
+            return this.pattern.matcher(textToCheck).results().findAny().isPresent();
+        } else {
+            return this.trie.containsMatch(textToCheck);
+        }
+
+    }
+
+    public List<MatchPosition> getMatches(String text) {
+        String textToCheck = text;
+        if (this.values.isEmpty()) {
+            return new ArrayList<>();
+        }
+        if (this.pattern != null) {
+            if (ignoreCase) {
+                textToCheck = textToCheck.toLowerCase(Locale.ROOT);
+            }
+            return this.pattern.matcher(textToCheck).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList());
+        } else {
+            return this.trie.parseText(textToCheck).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList());
+        }
+    }
+
+    @Data
+    @AllArgsConstructor
+    public static class MatchPosition {
+        private int startIndex;
+        private int endIndex;
+    }
+}
@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.util.Set;
+import java.util.regex.Pattern;
+
+public class SeparatorUtils {
+
+    private final static Set<Character> quotes = Set.of('\'', '\u0022', '\u00AB', '\u00BB', '\u2018', '\u2019', '\u201A', '\u201C', '\u201D', '\u201E'
+            , '\u2039', '\u203A');
+
+
+    public static boolean isSeparator(char c) {
+        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || quotes.contains(c);
+    }
+}
@@ -31,10 +31,12 @@ public class ImageService {
    public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId){

        var imageClassificationStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
+
+
        ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class);

        Map<Integer, List<PdfImage>> images = new HashMap<>();
-        imageServiceResponse.getImageMetadata().stream().forEach(imageMetadata -> {
+        imageServiceResponse.getData().stream().forEach(imageMetadata -> {
            var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)) : ImageType.OTHER;
            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), classification, imageMetadata.getPosition().getPageNumber()));
@@ -1,8 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.settings;

-import lombok.Data;
 import org.springframework.boot.context.properties.ConfigurationProperties;

+import lombok.Data;
+
@Data
@ConfigurationProperties("redaction-service")
 public class RedactionServiceSettings {
@@ -17,6 +18,8 @@ public class RedactionServiceSettings {

    private int analysisVersion = 1;

-    private boolean enableEntityRecognition = true;
+    private boolean nerServiceEnabled = true;
+
+    private boolean priorityMode;

 }
@@ -8,6 +8,7 @@ import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
@@ -43,6 +44,30 @@ public class RedactionStorageService {
        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any));
    }

+    @SneakyThrows
+    public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream){
+
+        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
+    }
+
+
+    public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
+
+        InputStreamResource inputStreamResource;
+        try {
+            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Imported redactions not available.");
+            return null;
+        }
+
+        try {
+            return objectMapper.readValue(inputStreamResource.getInputStream(), ImportedRedactions.class);
+        } catch (IOException e) {
+            throw new RuntimeException("Could not imported redactions", e);
+        }
+    }
+

    public RedactionLog getRedactionLog(String dossierId, String fileId) {

@@ -86,14 +111,13 @@ public class RedactionStorageService {
        try {
            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
        } catch (StorageObjectDoesNotExist e) {
-            log.debug("NER Entities not available.");
-            return null;
+            throw new NotFoundException("NER Entities are not available.");
        }

        try {
            return objectMapper.readValue(inputStreamResource.getInputStream(), NerEntities.class);
        } catch (IOException e) {
-            throw new RuntimeException("Could not convert NerEntities", e);
+            throw new RuntimeException("Could not convert NER Entities", e);
        }
    }

@@ -127,8 +151,6 @@ public class RedactionStorageService {
            return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
        }

-
-
    }

 }
@@ -9,6 +9,8 @@ server:
  port: 8080

 spring:
+  main:
+    allow-circular-references: true # FIXME
  profiles:
    active: kubernetes
  rabbitmq:
@@ -0,0 +1,115 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;
+
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class DictionaryServiceTest {
+
+    @MockBean
+    protected KieContainer kieContainer;
+
+    @MockBean
+    protected DictionaryClient dictionaryClient;
+
+    @Autowired
+    protected DictionaryService dictionaryService;
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+    @Test
+    public void testDictionaryServiceConsistency() {
+
+        when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
+        when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", "#cccccc",
+                "#cccccc", "#cccccc", "#cccccc", "#cccccc", "#cccccc",
+                "#cccccc", "#cccccc", "#cccccc", "#cccccc"));
+
+        var type1 = new Type();
+        type1.setType("type1");
+        type1.setId("type1");
+        type1.setVersion(1L);
+        type1.setHexColor("#cccccc");
+        type1.setHasDictionary(true);
+        type1.setEntries(Stream.of("a", "b", "c").map(t -> new DictionaryEntry(1, t, 1L, false, "type1")).collect(Collectors.toList()));
+
+
+        var type2 = new Type();
+        type2.setType("type2");
+        type2.setId("type2");
+        type2.setVersion(1L);
+        type2.setHexColor("#cccccc");
+        type2.setHasDictionary(true);
+        type2.setEntries(Stream.of("d", "e", "f").map(t -> new DictionaryEntry(1, t, 1L, false, "type2")).collect(Collectors.toList()));
+
+
+        var type1Updated = new Type();
+        type1Updated.setType("type1");
+        type1Updated.setId("type1");
+        type1Updated.setVersion(2L);
+        type1Updated.setHexColor("#cccccc");
+        type1Updated.setHasDictionary(true);
+        type1Updated.setEntries(Stream.of("z", "q", "x").map(t -> new DictionaryEntry(1, t, 2L, false, "type1")).collect(Collectors.toList()));
+
+
+        when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1, type2));
+        when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(type1);
+        when(dictionaryClient.getDictionaryForType("type2", null)).thenReturn(type2);
+        when(dictionaryClient.getDictionaryForType("type1", 0L)).thenReturn(type1);
+        when(dictionaryClient.getDictionaryForType("type2", 0L)).thenReturn(type2);
+
+        dictionaryService.updateDictionary("dossierTemplateId", null);
+        var increments = dictionaryService.getDictionaryIncrements("dossierTemplateId",
+                DictionaryVersion.builder().dossierTemplateVersion(0L).build(), null);
+
+
+        when(dictionaryClient.getVersion(any())).thenReturn(1L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1Updated));
+        when(dictionaryClient.getDictionaryForType("type1", 1L)).thenReturn(type1Updated);
+
+        dictionaryService.updateDictionary("dossierTemplateId", null);
+        var updatedIncrements = dictionaryService.getDictionaryIncrements("dossierTemplateId",
+                DictionaryVersion.builder().dossierTemplateVersion(0L).build(), null);
+
+    }
+}
@@ -10,15 +10,14 @@ import org.springframework.core.io.InputStreamResource;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.InputStream;
+import java.util.*;

-public class FileSystemBackedStorageService extends S3StorageService {
+public class FileSystemBackedStorageService implements StorageService{

    private final Map<String, File> dataMap = new HashMap<>();

    public FileSystemBackedStorageService() {
-        super(null, null);
    }

    @SneakyThrows
@@ -33,16 +32,45 @@ public class FileSystemBackedStorageService extends S3StorageService {

    }

+    @Override
+    public void deleteObject(String objectId) {
+        dataMap.remove(objectId);
+    }
+
+    @Override
+    public boolean objectExists(String objectId) {
+        return dataMap.containsKey(objectId);
+    }
+
+    @Override
+    public void init() {
+
+    }
+
+    public List<String> listPaths(){
+        return new ArrayList<>(dataMap.keySet());
+    }
+
    @SneakyThrows
    @Override
    public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("test", ".tmp");
+        File tempFile = File.createTempFile("storage", objectId.replace("/","-"));

        IOUtils.write(data, new FileOutputStream(tempFile));

        dataMap.put(objectId, tempFile);
    }

+    @Override
+    @SneakyThrows
+    public void storeObject(String objectId, InputStream stream) {
+        File tempFile = File.createTempFile("test", ".tmp");
+
+        IOUtils.copy(stream, new FileOutputStream(tempFile));
+
+        dataMap.put(objectId, tempFile);
+    }
+
    public void clearStorage() {
        this.dataMap.forEach((k, v) -> {
            v.delete();
@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.model;
+package com.iqser.red.service.redaction.v1.server.annotate;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.model;
+package com.iqser.red.service.redaction.v1.server.annotate;

 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -1,8 +1,15 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.annotate;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import lombok.RequiredArgsConstructor;
+import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -15,21 +22,50 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
 import org.springframework.stereotype.Service;

-import java.awt.Color;
+import java.awt.*;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;

@Service
@RequiredArgsConstructor
 public class AnnotationService {

+    private final RedactionStorageService redactionStorageService;
+    private final DictionaryService dictionaryService;

-    public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
+    private final RedactionLogMergeService redactionLogMergeService;
+
+    public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
+
+        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
+        var mergedRedactionLog = redactionLogMergeService.provideRedactionLog(RedactionRequest.builder()
+                .fileId(annotateRequest.getFileId())
+                .manualRedactions(annotateRequest.getManualRedactions())
+                .dossierId(annotateRequest.getDossierId())
+                .dossierTemplateId(annotateRequest.getDossierTemplateId())
+                .build());
+        var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
+
+        try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
+            pdDocument.setAllSecurityToBeRemoved(true);
+
+            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
+            annotate(pdDocument, mergedRedactionLog, sectionsGrid);
+
+            try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
+                pdDocument.save(byteArrayOutputStream);
+                return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
+            }
+
+        } catch (Exception e) {
+            throw new RedactionException(e);
+        }
+    }
+
+    private void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {

        Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);

@@ -107,7 +143,7 @@ public class AnnotationService {

    private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {

-        if (redactionLogEntry.isManual()) {
+        if (redactionLogEntry.isLocalManualRedaction()) {
            return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\"";
        }
        return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry
@@ -0,0 +1,87 @@
+package com.iqser.red.service.redaction.v1.server.realdata;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.MessageType;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+
+import java.time.OffsetDateTime;
+import java.util.List;
+import java.util.Set;
+
+@Slf4j
+public class AnalyseFileRealDataIntegrationTest extends LiveDataIntegrationTest {
+
+    public static final String FILE_NAME = "test-file";
+
+    @Test
+    @SneakyThrows
+    public void testFile() {
+
+        ObjectMapper om = new ObjectMapper();
+        om.registerModule(new JavaTimeModule());
+
+        var file = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".pdf").getInputStream();
+        redactionStorageService.storeObject("dossierId", "fileId", FileType.ORIGIN, file);
+
+
+        try {
+            var nerData = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".ner.json").getInputStream();
+            redactionStorageService.storeObject("dossierId", "fileId", FileType.NER_ENTITIES, nerData);
+        } catch (Exception e) {
+            log.warn("No NER File Provided");
+            redactionServiceSettings.setNerServiceEnabled(false);
+        }
+
+
+        AnalyzeRequest ar = AnalyzeRequest.builder()
+                .fileId("fileId")
+                .dossierId("dossierId")
+                .analysisNumber(1)
+                .dossierTemplateId("dossierTemplateId")
+                .lastProcessed(OffsetDateTime.now())
+                .excludedPages(Set.of())
+                .fileAttributes(List.of())
+                .build();
+
+
+        try {
+            var text = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".text.json").getInputStream();
+            var sectionText = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".section-grid.json").getInputStream();
+            redactionStorageService.storeObject("dossierId", "fileId", FileType.TEXT, text);
+            redactionStorageService.storeObject("dossierId", "fileId", FileType.SECTION_GRID, sectionText);
+        } catch (Exception e) {
+            log.info("No text file provided, Performing Structure analysis");
+
+            ar.setMessageType(MessageType.STRUCTURE_ANALYSE);
+            String in = om.writeValueAsString(ar);
+            redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+        }
+
+
+        try {
+            var redactionLog = new ClassPathResource(BASE_DIR + "data/" + FILE_NAME + ".redaction-log.json").getInputStream();
+        } catch (Exception e) {
+            log.info("No redaction log provided, Performing full analysis");
+
+            ar.setMessageType(MessageType.ANALYSE);
+            String in = om.writeValueAsString(ar);
+            redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+        }
+
+
+        simulateIncrement(List.of("Desiree"), "PII", 3L);
+        ar.setMessageType(MessageType.REANALYSE);
+        String in = om.writeValueAsString(ar);
+        redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+
+
+        // var log = redactionStorageService.getRedactionLog("dossierId","fileId");
+    }
+
+}
@@ -0,0 +1,196 @@
+package com.iqser.red.service.redaction.v1.server.realdata;
+
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.*;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.ResourcePatternResolver;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class LiveDataIntegrationTest {
+
+    protected static String BASE_DIR = "performance/";
+
+    protected static String EFSA_SANITISATION_GFL_V1 = "dictionaries/EFSA_sanitisation_GFL_v1/";
+
+    @MockBean
+    protected DictionaryClient dictionaryClient;
+
+    @MockBean
+    protected EntityRecognitionClient entityRecognitionClient;
+
+    @MockBean
+    private FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
+
+    @MockBean
+    private LegalBasisClient legalBasisClient;
+
+    @Autowired
+    private ResourcePatternResolver resourcePatternResolver;
+
+    @MockBean
+    private RulesClient rulesClient;
+
+    @Autowired
+    protected DictionaryService dictionaryService;
+
+    @Autowired
+    protected RedactionStorageService redactionStorageService;
+
+    @Autowired
+    protected RedactionMessageReceiver redactionMessageReceiver;
+
+    @Autowired
+    protected FileSystemBackedStorageService fileSystemBackedStorageService;
+
+    @Autowired
+    protected RedactionServiceSettings redactionServiceSettings;
+
+    private List<Type> types;
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+
+    @SneakyThrows
+    @Before
+    public void prepareTest() {
+
+        when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
+        when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);
+
+        var rules = IOUtils.toString(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "rules.drl").getInputStream());
+        when(rulesClient.getRules(any())).thenReturn(JSONPrimitive.of(rules));
+
+        ObjectMapper objectMapper = new ObjectMapper();
+
+        var jsonNode = objectMapper.readTree(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "types.json").getInputStream());
+        types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(
+                new TypeReference<List<Type>>() {
+                }));
+
+        types.forEach(t -> {
+            t.setId(t.getType());
+            t.setVersion(0L);
+        });
+
+        when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(types);
+        when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>());
+
+
+        when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(), Colors.class));
+
+        when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
+            String typeName = answer.getArgument(0);
+
+            var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
+            if(found.isPresent()) {
+                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
+                type.setEntries(getEntries(typeName, type.getTypeId()));
+
+                return type;
+            }else{
+                return null;
+            }
+        });
+
+        dictionaryService.updateDictionary("dossierTemplateId","dossierId");
+    }
+
+    public void simulateIncrement(List<String> values, String deltaTypeName, long version) {
+        when(dictionaryClient.getVersion(anyString())).thenReturn(version);
+        when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(version);
+
+        when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
+            String typeName = answer.getArgument(0);
+
+            var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
+            if(found.isPresent()) {
+
+                var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
+
+                if (deltaTypeName.equalsIgnoreCase(typeName)) {
+                    type.setEntries(values.stream().map(l -> new DictionaryEntry(0, l, version, false, type.getTypeId())).collect(Collectors.toList()));
+                } else {
+                    type.setEntries(new ArrayList<>());
+                }
+
+                return type;
+            }else{
+                return null;
+            }
+        });
+    }
+
+
+    @Test
+    public void testUpdateDictionary() {
+        dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
+
+        var dict = dictionaryService.getDeepCopyDictionary("dossierTemplateId", "dossierId");
+        assertThat(dict.getLocalAccessMap().size()).isEqualTo(12);
+    }
+
+    @SneakyThrows
+    private List<DictionaryEntry> getEntries(String typeName, String typeId) {
+        Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
+        for (var resource : dictionaryResources) {
+            if (resource.getFilename().contains(typeName)) {
+
+                List<String> lines = IOUtils.readLines(resource.getInputStream());
+                return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
+
+            }
+        }
+        return new ArrayList<>();
+    }
+}
@@ -9,6 +9,7 @@ import org.junit.Test;

 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;

 public class EntitySearchUtilsTest {

@@ -16,8 +17,8 @@ public class EntitySearchUtilsTest {
    public void testNestedEntitiesRemoval() {

        Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        entities.add(nested);
        entities.add(nesting);
        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
@@ -39,14 +40,14 @@ public class EntitySearchUtilsTest {

        // Arrange
        Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        existingEntities.add(existingEntity1);
        existingEntities.add(existingEntity2);

        Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
-        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        foundEntities.add(foundEntities1);
        foundEntities.add(foundEntities2);

@@ -72,14 +73,14 @@ public class EntitySearchUtilsTest {

        // Arrange
        Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        existingEntities.add(existingEntity1);
        existingEntities.add(existingEntity2);

        Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
-        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        foundEntities.add(foundEntities1);
        foundEntities.add(foundEntities2);

@@ -104,14 +105,14 @@ public class EntitySearchUtilsTest {

        // Arrange
        Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE,EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        existingEntities.add(existingEntity1);
        existingEntities.add(existingEntity2);

        Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
-        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        foundEntities.add(foundEntities1);
        foundEntities.add(foundEntities2);

@@ -136,22 +137,25 @@ public class EntitySearchUtilsTest {

        // Arrange
        Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        existingEntities.add(existingEntity1);
        existingEntities.add(existingEntity2);

        Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        foundEntities.add(foundEntities1);
        foundEntities.add(foundEntities2);
+        foundEntities.add(foundEntities3);

        // Act
        Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);

        // Assert
-        assertThat(result.size()).isEqualTo(0);
+        assertThat(result.size()).isEqualTo(1);
+        assertThat(result.contains(foundEntities2));

    }

@@ -166,22 +170,27 @@ public class EntitySearchUtilsTest {

        // Arrange
        Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
        existingEntities.add(existingEntity1);
        existingEntities.add(existingEntity2);

        Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE);
-        foundEntities.add(foundEntities1);
-        foundEntities.add(foundEntities2);
+        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        foundEntities.add(foundEntitiesOverlap1);
+        foundEntities.add(foundEntitiesOverlap2);
+        foundEntities.add(foundEntitiesSubset1);
+        foundEntities.add(foundEntitiesSubset2);

        // Act
        Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);

        // Assert
-        assertThat(result.size()).isEqualTo(0);
+        assertThat(result.size()).isEqualTo(2);
+        assertThat(result).containsExactlyInAnyOrder(foundEntitiesSubset1, foundEntitiesSubset2);

    }

@@ -94,7 +94,7 @@ public class PdfSegmentationServiceTest {
        ImageServiceResponse imageServiceResponse = objectMapper.readValue(responseJson.getInputStream(), ImageServiceResponse.class);

        Map<Integer, List<PdfImage>> images = new HashMap<>();
-        imageServiceResponse.getImageMetadata().stream().forEach(imageMetadata -> {
+        imageServiceResponse.getData().stream().forEach(imageMetadata -> {
            images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
                    .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)), imageMetadata.getPosition().getPageNumber()));
        });
@@ -9,6 +9,7 @@ ribbon:
 spring:
  main:
    allow-bean-definition-overriding: true
+    allow-circular-references: true # FIXME

 processing.kafkastreams: false

@@ -17,7 +18,7 @@ platform.multi-tenancy:

 redaction-service:
  enable-image-classification: false
-  enable-entity-recognition: false
+  enable-entity-recognition: true

 storage:
  signer-type: 'AWSS3V4SignerType'
@@ -8620,4 +8620,6 @@ Doe
 M. Mustermann
 F. Lastname
 Mustermann
-Lastname
+Lastname
+Bojangles
+Tambourine Man
@@ -3,3 +3,9 @@ Report Number: 33168
 Page
 Report Number: BFI0714
 Tesh Consultants International
+B. Rahim
+C. J. Alfred
+Naka-27 Aomachi, Nomi, Ishikawa 923-1101, Japan, JP
+Özgür U. Reyhan
+Sude Halide Nurullah
+Xinyi Y. Tao
@@ -0,0 +1 @@
+C. J. Alfred Xinyi
@@ -1,239 +0,0 @@
-(Parent
-ADME - Bioanalyses, Vergèze,
-Aeration: Gentle
-Andrews mean
-Andrews pKi
-BASF study code
-Billington –
-Bond elute
-Brown Liquid
-Brown Orange
-Brown Solid
-Brown disco
-Brown discoloured
-Brown fat
-Brown liquid
-Brown solid
-Buehler (1980),
-Buehler (1980).
-Buehler 9
-Buehler Test.
-Buehler assay
-Buehler test
-Buehler)
-Buehler-like :
-Calandra lark
-Cary, NC,
-Chambersas for
-Class 4
-Class III’
-Class III’)
-Class method,
-Class2 (effectively
-ClassMethod) Vischim
-Clay =
-Cochran-Armitage Test.
-Cochran-armitage test.
-Dunnett 2
-Dunnett's "t''
-Dunnett's Test
-Dunnett's multiple
-Dunnett's t-test
-Dunnett's test
-Dunnett) **
-Dunnett) Dose
-Dunnett) Food
-Dunnett) Lower
-Dunnett, 1964)
-Dunnett’s Multiple
-Dunnett’s T
-Dunnett’s Test)
-Dunnett’s adjustment
-Dunnett’s multiple
-Dunnett’s post
-Dunnett’s t-test)
-Fine vacuolation
-Fine white
-Finewhite powder
-Fischer 344
-Fischer 344)
-Fischer F344
-Fischer rat
-Fischer) Contaminants:
-Fischer-344 acceptability
-Fischer-344/
-Fischer-344/DuCrj acceptability
-Fischer344 rats.
-Fischer344) :
-Fischerrat embryo
-Fischer’s exact
-Fisher 344
-Fisher Exact Test
-Fisher Exact test
-Fisher exact test
-Fisher's Exact
-Fisher-344 rat
-Fisher344 (F344/
-Fisher344 rat
-Fisher344 rats
-Fisher344 rats.
-Fisher344 rats/
-Fisher344) group
-Fisher’s exact.
-Freeman-Tukey transform
-Gentle aeration
-Grade 2:
-Grade 3)
-Grade 3:
-Green
-Green Alga
-Green as
-Green-, blue-,
-GreenAlga, Selenastrum
-GreenAlga.. Springborn
-Gross External
-Gross Necropsy
-Gross and
-Gross necropsies
-Gross necropsy
-Gross pathology
-Gross pathology:
-Hall Farm
-Hall Farm)
-HallFarm (396
-Ham's F12
-Ham’s F10
-Hand-held
-HarlanWinkelmann, Borchem,
-Hill International,
-Hill Top
-Hill criteria
-Hill, Bracknell,
-HillTop Chamber,
-Hillmodel was
-Hill’s model
-Japanese White
-Jerneplaque assay
-Kent. Blood
-Klimisch score
-Litchfield and Wilcoxon
-Long term
-Long-Term Toxicity
-Long-term
-Longbw/
-Major Macroscopic findings
-Major biliary
-Major defects
-Major deviations
-Major fetal
-Major metabolic
-Major metabolites
-Major route
-Major treatment-related
-Mallard (Anas
-Mallard with
-Manual-Hand held
-Marsh Frog
-Marsh harrier
-Masonjars (500
-Meier formula)
-Meier survival
-Miles anti-Canine
-Miles anti-dog
-Netherlands, 2011
-Netherlands, published:
-Netherlands. Report
-Penn State
-Porewater:
-Rattray study
-Read across
-Reed Bunting
-Reichert-Jung Cryo
-RifconGmbH. Oxon
-Ritz and
-S.P.-U/
-San Diego,
-Shaw's Farm,
-Sprague
-Strain JapaneseWhite
-Strong morphological
-Weight -
-Weight 2744
-Weight Gain
-Weight On
-Weight Range
-Weight and
-Weight and length
-Weight at
-Weight gain
-Weight loss
-Weight of
-Weight only
-Weight range
-Weight, age
-Weight, feed
-Weight-of-evidence approach
-Weight:
-Weight; Adult
-Weight; Whole
-WeightGain 0-1
-WeightRange:
-Weightat dosing:
-Weightrange 200-250g.
-White (Tif:
-White 10
-White 3
-White Age
-White Powder
-White Strain
-White blood
-White cell
-White crystalline
-White guideline
-White guinea-pigs
-White hygroscopic
-White powder
-White solid
-White stork
-White swan
-White to off-white
-White wagtail
-White, solid
-WhitePowder Lot/
-Whiteguideline :
-Whitehygroscopic solid
-Whitepowder Lot/
-Whitepowder Stability
-Whitesolid Batch
-Whiteswan)
-Whiteto off-white
-Wildlife International specifications
-Wildlife Internationalspecifications Water:
-Williams E
-Williams medium
-Williams' medium
-WilliamsE medium
-Williams’ test
-Wilson's method.
-Wilson. All foetal
-Wilson.All foetal
-Wood pigeons
-York.
-Zeneca dossier);
-Zenecadossier); see
-green algae
-CTL
-No details reported
-Not given
-Not indicated
-Not reported
-Not reported.
-Not stated
-Syngenta
-Syngenta AG
-N/A
-No details reported
-Not available
-Test facility
-TBD
-David Chubb
@@ -9,10 +9,9 @@ global Section section

 //rule "0: Expand CBI Authors with firstname initials"
 //    when
-//        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+//        Section(matchesType("CBI_author"))
 //    then
 //        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
-//        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
 //    end


@@ -20,22 +19,38 @@ rule "0: Add CBI_author from ai"
    when
        Section(aiMatchesType("CBI_author"))
    then
-        section.addAiEntities("CBI_author", "recommendation_CBI_author");
+        section.addAiEntities("CBI_author", "CBI_author");
    end

 rule "0: Combine ai types CBI_author from ai"
    when
        Section(aiMatchesType("ORG"))
    then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "recommendation_CBI_address", 3, false);
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
    end

 rule "0: Expand CBI Authors with firstname initials"
    when
-        Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
+        Section(matchesType("CBI_author"))
    then
        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
-        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+    end
+
+rule "0: Expand CBI_author and PII matches with salutation prefix"
+    when
+        Section((matchesType("CBI_author") || matchesType("PII")) && (
+            searchText.contains("Mr")
+            || searchText.contains("Mrs")
+            || searchText.contains("Ms")
+            || searchText.contains("Miss")
+            || searchText.contains("Sir")
+            || searchText.contains("Madam")
+            || searchText.contains("Madame")
+            || searchText.contains("Mme")
+        ))
+    then
+        section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
+        section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
    end


@@ -0,0 +1 @@
+{"importedRedactions":{"1":[{"id":"755273b5ddc6933eff7f6b6e055c803f","positions":[{"topLeft":{"x":275.0,"y":520.504},"width":55.675,"height":15.582,"page":1}]},{"id":"5e517d2c890272fc140578a26971b459","positions":[{"topLeft":{"x":164.0,"y":316.492},"width":50.7,"height":15.658,"page":1},{"topLeft":{"x":127.0,"y":302.613},"width":119.675,"height":14.902,"page":1},{"topLeft":{"x":81.0,"y":289.622},"width":210.025,"height":14.846,"page":1},{"topLeft":{"x":136.0,"y":275.912},"width":94.2,"height":13.033,"page":1},{"topLeft":{"x":158.0,"y":261.402},"width":52.793,"height":16.218,"page":1},{"topLeft":{"x":177.0,"y":247.413},"width":15.507,"height":16.148,"page":1}]}]}}
@@ -0,0 +1,872 @@
+{
+  "result": {
+    "1": [
+      {
+        "value": "Lastname, J.",
+        "startOffset": 54,
+        "endOffset": 66,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Doe, M.",
+        "startOffset": 67,
+        "endOffset": 74,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Mustermann Lastname M.",
+        "startOffset": 75,
+        "endOffset": 97,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Doe J. Mustermann M.",
+        "startOffset": 99,
+        "endOffset": 119,
+        "type": "CBI_author"
+      }
+    ],
+    "2": [
+      {
+        "value": "Eikenboom Charalampos",
+        "startOffset": 148,
+        "endOffset": 169,
+        "type": "ORG"
+      },
+      {
+        "value": "Schenk Tanja Schmitt ←",
+        "startOffset": 170,
+        "endOffset": 192,
+        "type": "ORG"
+      }
+    ],
+    "3": [
+      {
+        "value": "Rue Jean Baffier",
+        "startOffset": 214,
+        "endOffset": 230,
+        "type": "CBI_author"
+      },
+      {
+        "value": "7232",
+        "startOffset": 155,
+        "endOffset": 159,
+        "type": "POSTAL"
+      },
+      {
+        "value": "CX",
+        "startOffset": 160,
+        "endOffset": 162,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Warnsveld",
+        "startOffset": 163,
+        "endOffset": 172,
+        "type": "CITY"
+      },
+      {
+        "value": "Netherlands",
+        "startOffset": 174,
+        "endOffset": 185,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "NL",
+        "startOffset": 187,
+        "endOffset": 189,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Institut Industries",
+        "startOffset": 190,
+        "endOffset": 209,
+        "type": "ORG"
+      },
+      {
+        "value": "33",
+        "startOffset": 211,
+        "endOffset": 213,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Rue Jean Baffier",
+        "startOffset": 214,
+        "endOffset": 230,
+        "type": "STREET"
+      },
+      {
+        "value": "18000",
+        "startOffset": 232,
+        "endOffset": 237,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Bourges",
+        "startOffset": 238,
+        "endOffset": 245,
+        "type": "CITY"
+      },
+      {
+        "value": "France",
+        "startOffset": 247,
+        "endOffset": 253,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "18300",
+        "startOffset": 282,
+        "endOffset": 287,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Saint-Satur",
+        "startOffset": 288,
+        "endOffset": 299,
+        "type": "CITY"
+      },
+      {
+        "value": "France",
+        "startOffset": 301,
+        "endOffset": 307,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Lesdo Industries",
+        "startOffset": 312,
+        "endOffset": 328,
+        "type": "ORG"
+      },
+      {
+        "value": "Chäppelisträssli",
+        "startOffset": 330,
+        "endOffset": 346,
+        "type": "ORG"
+      },
+      {
+        "value": "6078",
+        "startOffset": 348,
+        "endOffset": 352,
+        "type": "POSTAL"
+      },
+      {
+        "value": "Lungern",
+        "startOffset": 353,
+        "endOffset": 360,
+        "type": "STREET"
+      },
+      {
+        "value": "Switzerland",
+        "startOffset": 362,
+        "endOffset": 373,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Shlissel'burgskaya Ulitsa",
+        "startOffset": 374,
+        "endOffset": 399,
+        "type": "ORG"
+      },
+      {
+        "value": "Nizhny Novgorod Oblast",
+        "startOffset": 401,
+        "endOffset": 423,
+        "type": "STREET"
+      },
+      {
+        "value": "Russia",
+        "startOffset": 425,
+        "endOffset": 431,
+        "type": "CITY"
+      },
+      {
+        "value": "603034",
+        "startOffset": 433,
+        "endOffset": 439,
+        "type": "POSTAL"
+      },
+      {
+        "value": "RU",
+        "startOffset": 441,
+        "endOffset": 443,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Karl Johans Gate",
+        "startOffset": 444,
+        "endOffset": 460,
+        "type": "STREET"
+      },
+      {
+        "value": "11",
+        "startOffset": 461,
+        "endOffset": 463,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "0154",
+        "startOffset": 465,
+        "endOffset": 469,
+        "type": "POSTAL"
+      },
+      {
+        "value": "Oslo",
+        "startOffset": 470,
+        "endOffset": 474,
+        "type": "CITY"
+      },
+      {
+        "value": "Norway",
+        "startOffset": 476,
+        "endOffset": 482,
+        "type": "COUNTRY"
+      }
+    ],
+    "4": [
+      {
+        "value": "Expand",
+        "startOffset": 67,
+        "endOffset": 73,
+        "type": "STATE"
+      },
+      {
+        "value": "Hint Clarissa",
+        "startOffset": 77,
+        "endOffset": 90,
+        "type": "ORG"
+      },
+      {
+        "value": "Dict",
+        "startOffset": 114,
+        "endOffset": 118,
+        "type": "ORG"
+      },
+      {
+        "value": "Authors-Dict",
+        "startOffset": 171,
+        "endOffset": 183,
+        "type": "ORG"
+      }
+    ],
+    "6": [
+      {
+        "value": "Michael N.",
+        "startOffset": 7,
+        "endOffset": 17,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Weyland Industries",
+        "startOffset": 76,
+        "endOffset": 94,
+        "type": "ORG"
+      }
+    ],
+    "7": [
+      {
+        "value": "Funnarie B.",
+        "startOffset": 7,
+        "endOffset": 18,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Authentic Diagnostics",
+        "startOffset": 73,
+        "endOffset": 94,
+        "type": "ORG"
+      }
+    ],
+    "8": [
+      {
+        "value": "Feuer A.",
+        "startOffset": 6,
+        "endOffset": 14,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Tyrell Corporation",
+        "startOffset": 74,
+        "endOffset": 92,
+        "type": "ORG"
+      }
+    ],
+    "12": [
+      {
+        "value": "Melanie",
+        "startOffset": 292,
+        "endOffset": 299,
+        "type": "CBI_author"
+      }
+    ],
+    "13": [
+      {
+        "value": "Stark Industries",
+        "startOffset": 184,
+        "endOffset": 200,
+        "type": "ORG"
+      }
+    ],
+    "14": [
+      {
+        "value": "Omni Consumer Products Do",
+        "startOffset": 197,
+        "endOffset": 222,
+        "type": "ORG"
+      }
+    ],
+    "15": [
+      {
+        "value": "Omni Consumer Products Do",
+        "startOffset": 122,
+        "endOffset": 147,
+        "type": "ORG"
+      }
+    ],
+    "16": [
+      {
+        "value": "Asya Lyon",
+        "startOffset": 253,
+        "endOffset": 262,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Carina Madsen",
+        "startOffset": 264,
+        "endOffset": 277,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Alexandra Häusler",
+        "startOffset": 279,
+        "endOffset": 296,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Hanke Mendel",
+        "startOffset": 298,
+        "endOffset": 310,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Kwok, Jun K.",
+        "startOffset": 444,
+        "endOffset": 456,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Tu Wong",
+        "startOffset": 458,
+        "endOffset": 465,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Qiang Suen",
+        "startOffset": 467,
+        "endOffset": 477,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Zhou Mah",
+        "startOffset": 479,
+        "endOffset": 487,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Lei W. Huang",
+        "startOffset": 499,
+        "endOffset": 511,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Ru X.",
+        "startOffset": 513,
+        "endOffset": 518,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Oxford University Press",
+        "startOffset": 166,
+        "endOffset": 189,
+        "type": "ORG"
+      },
+      {
+        "value": "Iakovos Geiger",
+        "startOffset": 222,
+        "endOffset": 236,
+        "type": "ORG"
+      },
+      {
+        "value": "Julian Ritter",
+        "startOffset": 238,
+        "endOffset": 251,
+        "type": "CITY"
+      },
+      {
+        "value": "Asya Lyon",
+        "startOffset": 253,
+        "endOffset": 262,
+        "type": "ORG"
+      },
+      {
+        "value": "Carina Madsen",
+        "startOffset": 264,
+        "endOffset": 277,
+        "type": "CITY"
+      },
+      {
+        "value": "Alexandra Häusler",
+        "startOffset": 279,
+        "endOffset": 296,
+        "type": "ORG"
+      },
+      {
+        "value": "Hanke Mendel",
+        "startOffset": 298,
+        "endOffset": 310,
+        "type": "ORG"
+      },
+      {
+        "value": "Ranya",
+        "startOffset": 312,
+        "endOffset": 317,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Eikenboom",
+        "startOffset": 318,
+        "endOffset": 327,
+        "type": "ORG"
+      },
+      {
+        "value": "Min Kwok",
+        "startOffset": 440,
+        "endOffset": 448,
+        "type": "ORG"
+      },
+      {
+        "value": "Qiang Suen",
+        "startOffset": 467,
+        "endOffset": 477,
+        "type": "CITY"
+      },
+      {
+        "value": "Zhou Mah",
+        "startOffset": 479,
+        "endOffset": 487,
+        "type": "ORG"
+      },
+      {
+        "value": "Ning Liu",
+        "startOffset": 489,
+        "endOffset": 497,
+        "type": "STREET"
+      },
+      {
+        "value": "Lei W. Huang, Ru X. Wu",
+        "startOffset": 499,
+        "endOffset": 521,
+        "type": "ORG"
+      }
+    ],
+    "17": [
+      {
+        "value": "Nurullah Özgür",
+        "startOffset": 210,
+        "endOffset": 224,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Reyhan B.",
+        "startOffset": 228,
+        "endOffset": 237,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Alfred Xinyi Y.",
+        "startOffset": 250,
+        "endOffset": 265,
+        "type": "CBI_author"
+      },
+      {
+        "value": "Redacted",
+        "startOffset": 12,
+        "endOffset": 20,
+        "type": "ORG"
+      },
+      {
+        "value": "Aomachi",
+        "startOffset": 154,
+        "endOffset": 161,
+        "type": "ORG"
+      },
+      {
+        "value": "Nomi",
+        "startOffset": 163,
+        "endOffset": 167,
+        "type": "ORG"
+      },
+      {
+        "value": "Ishikawa",
+        "startOffset": 169,
+        "endOffset": 177,
+        "type": "CITY"
+      },
+      {
+        "value": "923-1101",
+        "startOffset": 178,
+        "endOffset": 186,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Japan",
+        "startOffset": 188,
+        "endOffset": 193,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "JP",
+        "startOffset": 195,
+        "endOffset": 197,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Sude Halide Nurullah Özgür U. Reyhan B. Rahim C. J. Alfred Xinyi Y. Tao Clara Siegfried",
+        "startOffset": 198,
+        "endOffset": 285,
+        "type": "ORG"
+      },
+      {
+        "value": "Dict",
+        "startOffset": 301,
+        "endOffset": 305,
+        "type": "ORG"
+      }
+    ],
+    "18": [
+      {
+        "value": "Redact Emails",
+        "startOffset": 12,
+        "endOffset": 25,
+        "type": "ORG"
+      }
+    ],
+    "19": [
+      {
+        "value": "Central Research Industry",
+        "startOffset": 274,
+        "endOffset": 299,
+        "type": "ORG"
+      },
+      {
+        "value": "Maximiliam Schmitt",
+        "startOffset": 530,
+        "endOffset": 548,
+        "type": "ORG"
+      },
+      {
+        "value": "European Central Institute",
+        "startOffset": 727,
+        "endOffset": 753,
+        "type": "ORG"
+      },
+      {
+        "value": "Emilia Lockhart Alternative",
+        "startOffset": 775,
+        "endOffset": 802,
+        "type": "ORG"
+      },
+      {
+        "value": "Cyberdyne Systems Tower",
+        "startOffset": 812,
+        "endOffset": 835,
+        "type": "ORG"
+      },
+      {
+        "value": "121a",
+        "startOffset": 844,
+        "endOffset": 848,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Hong Kong",
+        "startOffset": 849,
+        "endOffset": 858,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "BT",
+        "startOffset": 860,
+        "endOffset": 862,
+        "type": "COUNTRY"
+      }
+    ],
+    "21": [
+      {
+        "value": "Central Research Industry",
+        "startOffset": 270,
+        "endOffset": 295,
+        "type": "ORG"
+      },
+      {
+        "value": "Maximiliam Schmitt",
+        "startOffset": 526,
+        "endOffset": 544,
+        "type": "ORG"
+      },
+      {
+        "value": "European Central Institute",
+        "startOffset": 723,
+        "endOffset": 749,
+        "type": "ORG"
+      },
+      {
+        "value": "Emilia Lockhart Alternative",
+        "startOffset": 771,
+        "endOffset": 798,
+        "type": "ORG"
+      },
+      {
+        "value": "Cyberdyne Systems Tower",
+        "startOffset": 808,
+        "endOffset": 831,
+        "type": "ORG"
+      },
+      {
+        "value": "121a",
+        "startOffset": 840,
+        "endOffset": 844,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Hong Kong",
+        "startOffset": 845,
+        "endOffset": 854,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "BT",
+        "startOffset": 856,
+        "endOffset": 858,
+        "type": "COUNTRY"
+      }
+    ],
+    "22": [
+      {
+        "value": "Riddley",
+        "startOffset": 236,
+        "endOffset": 243,
+        "type": "ORG"
+      },
+      {
+        "value": "359-21",
+        "startOffset": 259,
+        "endOffset": 265,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Huam-dong",
+        "startOffset": 266,
+        "endOffset": 275,
+        "type": "STATE"
+      },
+      {
+        "value": "Yongsan-gu",
+        "startOffset": 276,
+        "endOffset": 286,
+        "type": "CITY"
+      },
+      {
+        "value": "Seoul",
+        "startOffset": 287,
+        "endOffset": 292,
+        "type": "CITY"
+      },
+      {
+        "value": "South Korea Phone",
+        "startOffset": 294,
+        "endOffset": 311,
+        "type": "COUNTRY"
+      },
+      {
+        "value": "Soylent Corporation",
+        "startOffset": 500,
+        "endOffset": 519,
+        "type": "ORG"
+      }
+    ],
+    "23": [
+      {
+        "value": "Umbrella Corporation",
+        "startOffset": 208,
+        "endOffset": 228,
+        "type": "ORG"
+      },
+      {
+        "value": "Jill",
+        "startOffset": 238,
+        "endOffset": 242,
+        "type": "ORG"
+      },
+      {
+        "value": "359-21",
+        "startOffset": 262,
+        "endOffset": 268,
+        "type": "CARDINAL"
+      },
+      {
+        "value": "Huam-dong",
+        "startOffset": 269,
+        "endOffset": 278,
+        "type": "STATE"
+      },
+      {
+        "value": "Yongsan-gu",
+        "startOffset": 279,
+        "endOffset": 289,
+        "type": "CITY"
+      },
+      {
+        "value": "Seoul",
+        "startOffset": 290,
+        "endOffset": 295,
+        "type": "CITY"
+      },
+      {
+        "value": "South Korea Phone",
+        "startOffset": 297,
+        "endOffset": 314,
+        "type": "COUNTRY"
+      }
+    ],
+    "26": [
+      {
+        "value": "Umbrella Corporation",
+        "startOffset": 209,
+        "endOffset": 229,
+        "type": "ORG"
+      }
+    ],
+    "28": [
+      {
+        "value": "Purity Hint",
+        "startOffset": 9,
+        "endOffset": 20,
+        "type": "ORG"
+      },
+      {
+        "value": "Hint",
+        "startOffset": 35,
+        "endOffset": 39,
+        "type": "ORG"
+      }
+    ],
+    "29": [
+      {
+        "value": "Ignore",
+        "startOffset": 9,
+        "endOffset": 15,
+        "type": "STREET"
+      }
+    ],
+    "30": [
+      {
+        "value": "Redact Signatures Redact",
+        "startOffset": 12,
+        "endOffset": 36,
+        "type": "ORG"
+      },
+      {
+        "value": "Dilara",
+        "startOffset": 139,
+        "endOffset": 145,
+        "type": "ORG"
+      },
+      {
+        "value": "Tobias Müller",
+        "startOffset": 197,
+        "endOffset": 210,
+        "type": "ORG"
+      }
+    ],
+    "31": [
+      {
+        "value": "Redact Logo Redact",
+        "startOffset": 9,
+        "endOffset": 27,
+        "type": "ORG"
+      }
+    ],
+    "41": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "42": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "43": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "44": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "45": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "46": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "47": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "48": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ],
+    "49": [
+      {
+        "value": "Page-Footer",
+        "startOffset": 10,
+        "endOffset": 21,
+        "type": "STREET"
+      }
+    ]
+  }
+}
@@ -1,7 +1,7 @@
 {
  "dossierId": "f889853e-4bf8-49a9-aae5-c38605c6ef40",
  "fileId": "22ef63e29bb2a27db8497272336f6b32",
-  "imageMetadata": [
+  "data": [
    {
      "classification": {
        "probabilities": {
@@ -0,0 +1,12 @@
+{
+  "result": {
+    "1": [
+      {
+        "value": "Mannheim",
+        "startOffset": 0,
+        "endOffset": 8,
+        "type": "CITY"
+      }
+    ]
+  }
+}
@@ -0,0 +1,12 @@
+{
+  "defaultColor": "#9398a0",
+  "requestAdd": "#04b093",
+  "requestRemove": "#04b093",
+  "notRedacted": "#c498fa",
+  "analysisColor": "#dd4d50",
+  "updatedColor": "#fdbd00",
+  "dictionaryRequestColor": "#5b97db",
+  "manualRedactionColor": "#9398a0",
+  "previewColor": "#9398a0",
+  "ignoredHintColor": "#e7d4ff"
+}
@@ -0,0 +1,21 @@
+5 Batch
+Author
+Batch Analysis
+Certificate of Analysis
+Chemical Investigator
+Contributors
+Five Batch
+GLP Certificate
+Pathologist
+Performing Laboratory
+Principle Investigator
+Quality Assurance
+References
+Report of Peer Review
+Study Director
+Study Monitor
+Test Facility
+Test Site
+Trial Site
+altitude
+latitude
@@ -0,0 +1,234 @@
+. Comparative Biochemistry and Physiology Part C: Toxicology & Pharmacology C Toxicology & Pharmacology
+. J. Exp. Zool
+. Lewis Publ., Boca Raton,
+1944; 82:377-390
+7th ed.; The Iowa State University Press: Ames, IA
+7th ed.; The Joint Nature ConservationIowa State University Press: Ames, IAReproduction, Fertility and Development 2
+ATLA
+Academic Press
+Advances in
+Ageing Dev.
+Agromedicine
+Am J Epidemiol
+Am J Ind Med
+Am J Med Genet A
+Am J Respir Crit Care Med
+American Society for Testing and Materials, Philadelphia, Pennsylvania.
+Anal Chem.
+Anal, Chem.,
+Animal Reproduction Science
+Annals of
+Annals of Maths Stats
+Apidologie
+Aq. Toxicol.
+Aquatic Ecology
+Aquatic Toxicology
+Aquatic toxicology
+Arch Environ Contam Toxicol
+Arch Toxicol
+Arch Toxicol.
+Archives of Environmental Contamination and Toxicology
+Archives of Toxicology
+Australasian Journal of Ecotoxicology
+BioFactors
+Biomedical chromatography
+Biometrics
+Biometrika
+Birth Defects Res. B. Dev. Reprod. Toxicol
+Birth Defects Res. B. Dev. Reprod. Toxicol.
+Br Med J
+British Health and Safety Executive, Chemicals Regulation Directorate Pesticides
+Brown Company Publishers, Dubuque, Iowa
+Bull Environ Contam Toxicol.
+Bull. Environ. Contam. Toxicol
+Bulletin
+Bulletin of Environmental Contamination and Toxicology
+CHEMOSPHERE
+Cahiers Agricultures
+Cancer Letters
+Cancer Res
+Carcinogenesis
+Chem. Res. Toxicol.
+Chem. Res. in Toxicol.
+Chemosphere
+Chinese Journal of Chromatography
+Clin Anat.
+Comparative Biochemistry and Physiology Part C: Toxicology & Pharmacology C Toxicology & Pharmacology
+Crit Rev Toxicol
+Critical Reviews in Toxicology
+Curr. Med. Chem.
+Current approaches in the statistical analysis of ecotoxicity data: guidance to application
+DEFRA
+Deutsche Lebensmittel-Rundschau
+Deutsche Lebensmittel-Rundschau.
+Dongbei Nongye Daxue Xuebao
+Dual-Gold
+EFSA Journal
+Ecological Research
+Ecotoxicol. Environ. Safety
+Ecotoxicology and Environmental Safety
+Endocrinology
+Env. Mol. Mutagen
+Environ Health Perspect
+Environ Health Perspect.
+Environ Health Perspect. 1
+Environ Mutat
+Environ Sci Pollut Res
+Environ Sci Technol
+Environ Toxicol.
+Environ and Molecular Mutagenesis
+Environ. Health Perspec.
+Environ. Health Perspect.
+Environ. Sci. Technol
+Environ. Sci. Technol.
+Environment International
+Environmental & Molecular Mutagenesis
+Environmental Health
+Environmental Health Perspectives
+Environmental Monitoring and Assessment
+Environmental Pollution
+Environmental Protection Agency Federal Register
+Environmental Science & Technology
+Environmental Science and Technology
+Environmental Science and Technology.
+Environmental Toxicology and Chemistry
+Environmental monitoring and assessment
+Environmental toxicology and chemistry
+Essays in Honor of Harold Hotelling
+Eur Respir
+Experimental Cel] Research
+FUNDAMENTAL AND APPLIED TOXICOLOGY
+Fischer, Jena
+Fish Sci
+Food Cosmet. Toxicol.
+Franckh, Stuttgar
+Franckh, Stuttgart
+Free Rad. Res
+Fundamentals of Applied Toxicology
+Fundamentals of Applied Toxicology1988
+Gen. Comp. Endocrinol
+Gen. Comp.Endocrinol
+Gen.Com. Endocrinol.
+General and Comparative Endocrinology
+Greenhouse Pest Management
+Gustav Fischer Verlag, Stuttgart
+Handbook of water purity and quality.
+Health and Safety Executive, Chemi-cals Regulation Directorate Pesticides
+Herpetologica
+High-Throughput Screening Methods in Toxicity Testing
+High-Throughput Screening Methods in Toxicity Testing. Hoboken, NJ: John Wiley & Sons
+ISBN
+ISSN
+ISSN/ISBN
+Ibis
+Int Arch Occup Environ Health
+Int J Cancer
+Irish Journal of Agricultural and Food Research
+Ist Annual Symposium on Aquatic Toxicology ASTM,
+J Econom
+J Endocrinol
+J LARGILLIERE
+J Radiat Res
+J Steroid Biochem Mol Biol
+J. Invest. Derm
+Joint Nature Conservation Committee
+Journal Pharmacology Experimental Therapeutics
+Journal of
+Journal of Animal Ecology
+Journal of Applied Ecology
+Journal of Applied Entomology
+Journal of Economic Entomology
+Journal of Environmental Science and Health
+Journal of Experimental Biology and Ecology
+Journal of Hazardous Materials
+Journal of Microbiology
+Journal of Toxicology and
+Journal of agricultural and food chemistry
+Journal of the American College of Toxicology
+Journal of the Royal Statistical Society,
+Leaflet
+Limnologica
+Marine Enironmental Research
+Marine Environmental Research
+Marine Pollution Bulletin
+Mass Spectrometry
+Medical Journal
+Middle Atlantic Reproduction and Teratology Association
+Mol Mutagen
+Mol. Cell. Endocrinol.
+Mutagenesis
+Mutat Res
+Mutation Research
+National Academy of Science,
+Nature
+Neurotoxicity research
+Nonparametric Statistics for the Behavioral Sciences
+OECD Guideline for the testing of chemicals
+OECD Guidelines for Testing of Chemicals
+OECD Guidelines for testing of chemicals.
+OECD Principles of Good Laboratory Practice and Compliance Monitoring
+OECD Series on Testing Assessment
+Occup Environ Med
+Occupational and Environmental Medicine
+Office of Prevention, Pesticides and Toxic Substances
+Official Journal of the European Communities
+Oxford University Press
+PLoS ONE
+Pediat. Res
+Pesticide Biochemistry and Physiology
+Physiol Chem
+Planta daninha
+Press
+Principles and Procedures of Statistics
+Principles and Procedures of Statistics, A Biometrical Approach
+Proc Natl Acad Sci USA
+Psychopharmacologia
+Publication
+Publications Office of the European Union, Luxembourg
+Published: Y
+Publisher Gustav Fischer, Stuttgart
+Raven press,
+Reg. Tox. Pharmacol
+Regul. Toxicol. Pharmacol
+Regulatory Toxicology and Pharmacology
+Reproductive BioMedicine Online
+Reproductive Toxicology
+Schweizerbart’sche Verlagsbuchhandlung
+Science of the Total Environment
+Scientific Reports
+Stain Technol
+Superorder Monogonnata
+Swiss Ordinance relating to Good Laboratory Practice
+Teratology
+The American Statistician
+The Chemical Society,
+The Inland Waters Volume
+The Royal Society for the Protection of Birds
+Tidepool Scientific Software, Mckinleyville, CA, USA
+Toxicol Chem
+Toxicol Pathol.
+Toxicol Sci
+Toxicol Sci.
+Toxicol Sci. 1
+Toxicol in Vitro
+Toxicol. Sci,
+Toxicol. Sci.
+Toxicol. in Vitro
+Toxicologic Pathology
+Toxicological Sciences
+Toxicological and Environmental Chemistry
+Toxicology Letters
+Toxicology and Applied Pharmacology
+Toxicology and Industrial Health
+Toxicology in Vitro
+Trends in Neurosciences
+Umweltbundesamt Development & Research Project
+WATER CHLORINATION:CHEM, ENVIRON. IMPACT HEALTH EFF., PROC. CONF
+Water Res.
+Workshop
+World Journal of Pharmaceutical Research
+XIII Symposium Pesticide Chemistry
+edition
+http://www.iobc-wprs.org
+ournal of the Royal Statistical Society
+paper was presented
@@ -0,0 +1,22 @@
+{
+  "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+  "name": "EFSA sanitisation GFL v1",
+  "description": "Redact all authors, laboratory addresses and other geolocation.",
+  "dateAdded": "2021-01-21T14:25:35Z",
+  "dateModified": "2021-07-19T08:09:27Z",
+  "createdBy": "e3aed6ea-a9e5-4f3f-bde7-6f0fe0c4362c",
+  "modifiedBy": "e3aed6ea-a9e5-4f3f-bde7-6f0fe0c4362c",
+  "validFrom": null,
+  "validTo": null,
+  "downloadFileTypes": [
+    "PREVIEW",
+    "REDACTED",
+    "DELTA_PREVIEW"
+  ],
+  "reportTypes": [
+    "EXCEL_MULTI_FILE",
+    "WORD_SINGLE_FILE_APPENDIX_A1_TEMPLATE",
+    "WORD_SINGLE_FILE_APPENDIX_A2_TEMPLATE"
+  ],
+  "dossierTemplateStatus": "ACTIVE"
+}
@@ -0,0 +1,47 @@
+[
+  {
+    "name": "1.1 personal data (incl. geolocation); Article 39(e)(3)",
+    "description": "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)",
+    "reason": "Article 39(e)(3) of Regulation (EC) No 178/2002"
+  },
+  {
+    "name": "1.2 vertebrate study related personal data (incl. geolocation); Article 39(e)(2)",
+    "description": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information",
+    "reason": "Article 39(e)(2) of Regulation (EC) No 178/2002"
+  },
+  {
+    "name": "2. manufacturing or production process",
+    "description": "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety",
+    "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"
+  },
+  {
+    "name": "3. links between a producer and applicant",
+    "description": "commercial links between a producer or importer and the applicant or the authorisation holder, where applicable",
+    "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No  178/2002)"
+  },
+  {
+    "name": "4. commercial information",
+    "description": "commercial information revealing sourcing, market shares or business strategy of the applicant",
+    "reason": "Article 63(2)(a) of Regulation (EC) No 1107/2009  (making reference to Article 39 of Regulation EC No 178/2002)"
+  },
+  {
+    "name": "5. quantitative composition",
+    "description": "quantitative composition of the subject matter of the request, except for information which is relevant to the assessment of safety",
+    "reason": "Article 63(2)(a) of Regulation (EC) No  1107/2009 (making reference to Article 39 of Regulation EC No 178/2002)"
+  },
+  {
+    "name": "6. specification of impurity",
+    "description": "the specification of impurity of the active substance and the related methods of analysis for impurities in the active substance as manufactured, except for the impurities that are considered to be toxicologically, ecotoxicologically or environmentally relevant and the related methods of analysis for such impurities",
+    "reason": "Article 63(2)(b) of Regulation (EC) No 1107/2009"
+  },
+  {
+    "name": "7. results of production batches",
+    "description": "results of production batches of the active substance including impurities",
+    "reason": "Article 63(2)(c) of Regulation (EC) No 1107/2009"
+  },
+  {
+    "name": "8. composition of a plant protection product",
+    "description": "information on the complete composition of a plant protection product",
+    "reason": "Article 63(2)(d) of Regulation (EC) No 1107/2009"
+  }
+]
@@ -0,0 +1,480 @@
+package drools
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+
+global Section section
+
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
+rule "0: Add CBI_author from ai"
+    when
+        Section(aiMatchesType("CBI_author"))
+    then
+        section.addAiEntities("CBI_author", "CBI_author");
+    end
+
+
+rule "0: Combine ai types CBI_author from ai"
+    when
+        Section(aiMatchesType("ORG"))
+    then
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+
+rule "1: Redact CBI Authors (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "2: Redact CBI Authors (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "3: Redact not CBI Address (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        section.ignoreRecommendations("CBI_address");
+    end
+
+
+rule "4: Redact CBI Address (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "5: Do not redact genitive CBI_author"
+    when
+        Section(matchesType("CBI_author"))
+    then
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+    end
+
+
+rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "13: Redact addresses that start with BL or CTL"
+    when
+        Section(searchText.contains("BL") || searchText.contains("CT"))
+    then
+        section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found");
+    end
+
+
+rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "16: Add recommendation for Addresses in Test Organism sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
+    then
+        section.recommendLineAfter("Source:", "CBI_address");
+    end
+
+
+rule "17: Add recommendation for Addresses in Test Animals sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
+    then
+        section.recommendLineAfter("Source", "CBI_address");
+    end
+
+
+rule "18: Do not redact Names and Addresses if Published Information found"
+    when
+        Section(matchesType("published_information"))
+    then
+        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
+        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
+    end
+
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+
+rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 19, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 20, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "21: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "22: Redact Emails by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "23: Redact telephone numbers by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
+    then
+        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "24: Redact telephone numbers by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
+    then
+        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "25: Recommend telephone numbers by RegEx"
+    when
+        Section(containsRegEx("\\d{3,}", true) || containsRegEx("[+]\\d{2,}", true))
+    then
+        section.addRecommendationByRegEx("([+]\\d{2,3} (\\d{7,12})\\b|((([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|(\\(?\\b(\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b))", true, 1, "PII");
+    end
+
+
+rule "26: Redact contact information (Non vertebrate study)"
+    when
+       Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+       || text.contains("Phone:")
+       || text.contains("Fax:")
+       || text.contains("Tel.:")
+       || text.contains("Tel:")
+       || text.contains("E-mail:")
+       || text.contains("Email:")
+       || text.contains("e-mail:")
+       || text.contains("E-mail address:")
+       || text.contains("Alternative contact:")
+       || text.contains("Telephone number:")
+       || text.contains("Telephone No:")
+       || text.contains("Fax number:")
+       || text.contains("Telephone:")
+       || text.contains("Phone No.")
+       || text.contains("European contact:")))
+    then
+          section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+     end
+
+
+rule "27: Redact contact information (Vertebrate study)"
+    when
+       Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+       || text.contains("Phone:")
+       || text.contains("Fax:")
+       || text.contains("Tel.:")
+       || text.contains("Tel:")
+       || text.contains("E-mail:")
+       || text.contains("Email:")
+       || text.contains("e-mail:")
+       || text.contains("E-mail address:")
+       || text.contains("Alternative contact:")
+       || text.contains("Telephone number:")
+       || text.contains("Telephone No:")
+       || text.contains("Fax number:")
+       || text.contains("Telephone:")
+       || text.contains("Phone No.")
+       || text.contains("European contact:")))
+    then
+          section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+     end
+
+
+rule "28: Redact contact information if applicant is found (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
+    then
+        section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "29: Redact contact information if applicant is found (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
+    then
+        section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "30: Redact contact information if Producer is found (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
+    then
+        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "31: Redact contact information if Producer is found (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
+    then
+        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "32: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "33: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "34: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "35: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "36: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactNot("CBI_address", 36, "Performing laboratory found for non vertebrate study");
+    end
+
+
+rule "37: Redact PERFORMING LABORATORY (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "50: Purity Hint"
+    when
+        Section(searchText.toLowerCase().contains("purity"))
+    then
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+rule "51: Ignore dossier_redaction entries if confidential"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction"));
+    then
+        section.ignore("dossier_redaction");
+    end
+
+
+rule "52: Redact signatures (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 52, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "53: Redact signatures (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 53, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "54: Redact Logos (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
+    then
+        section.redactImage("logo", 54, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
@@ -0,0 +1,185 @@
+{
+  "types": [
+    {
+      "type": "CBI_address",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 140,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "All site names and addresses, and location (e.g. Syngenta, Monthey, GPS Co-ordinates, Mr Smith of … providing the…). Except addresses in published literature and the applicant address.",
+      "addToDictionaryAction": true,
+      "label": "CBI Address",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "CBI_author",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 130,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "All authors named in the study documentation. Except names in published literature.",
+      "addToDictionaryAction": true,
+      "label": "CBI Author",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "PII",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 150,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "Not authors but listed in the document: Names, signatures, telephone, email etc.; e.g. Reg Manager, QA Manager",
+      "addToDictionaryAction": true,
+      "label": "PII",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "formula",
+      "hexColor": "#036ffc",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1002,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure formula colors.",
+      "addToDictionaryAction": false,
+      "label": "Formula",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "hint_only",
+      "hexColor": "#fa98f7",
+      "recommendationHexColor": "#8df06c",
+      "rank": 50,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Entries of this dictionary will be highlighted only",
+      "addToDictionaryAction": false,
+      "label": "Hint Only",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "image",
+      "hexColor": "#bdd6ff",
+      "recommendationHexColor": "#8df06c",
+      "rank": 999,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure image colors.",
+      "addToDictionaryAction": false,
+      "label": "Image",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "logo",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1001,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure logo colors.",
+      "addToDictionaryAction": false,
+      "label": "Logo",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "must_redact",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 100,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Entries of this dictionary get redacted wherever found.",
+      "addToDictionaryAction": false,
+      "label": "Must Redact",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "ocr",
+      "hexColor": "#bdd6ff",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1000,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure ocr colors.",
+      "addToDictionaryAction": false,
+      "label": "Ocr",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "published_information",
+      "hexColor": "#85ebff",
+      "recommendationHexColor": "#8df06c",
+      "rank": 70,
+      "hint": true,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "Manual managed list of public journals and papers that need no redaction",
+      "addToDictionaryAction": true,
+      "label": "Published Information",
+      "hasDictionary": true,
+      "systemManaged": false
+    },
+    {
+      "type": "signature",
+      "hexColor": "#9398a0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 1003,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": true,
+      "recommendation": false,
+      "description": "Empty dictionary used to configure signature colors.",
+      "addToDictionaryAction": false,
+      "label": "Signature",
+      "hasDictionary": false,
+      "systemManaged": true
+    },
+    {
+      "type": "imported_redaction",
+      "hexColor": "#f0f0c0",
+      "recommendationHexColor": "#8df06c",
+      "rank": 9999,
+      "hint": false,
+      "dossierTemplateId": "b959cf8a-e8f8-470b-aad4-6fc602ef110b",
+      "caseInsensitive": false,
+      "recommendation": false,
+      "description": "Redaction Annotations that were imported from documents",
+      "addToDictionaryAction": false,
+      "label": "Imported Redaction",
+      "hasDictionary": false,
+      "systemManaged": true,
+      "autoHideSkipped": true
+    }
+  ]
+}
				`@@ -0,0 +1 @@`
				{"importedRedactions":{"1":[{"id":"755273b5ddc6933eff7f6b6e055c803f","positions":[{"topLeft":{"x":275.0,"y":520.504},"width":55.675,"height":15.582,"page":1}]},{"id":"5e517d2c890272fc140578a26971b459","positions":[{"topLeft":{"x":164.0,"y":316.492},"width":50.7,"height":15.658,"page":1},{"topLeft":{"x":127.0,"y":302.613},"width":119.675,"height":14.902,"page":1},{"topLeft":{"x":81.0,"y":289.622},"width":210.025,"height":14.846,"page":1},{"topLeft":{"x":136.0,"y":275.912},"width":94.2,"height":13.033,"page":1},{"topLeft":{"x":158.0,"y":261.402},"width":52.793,"height":16.218,"page":1},{"topLeft":{"x":177.0,"y":247.413},"width":15.507,"height":16.148,"page":1}]}]}}