RM-76: Ignore excluded page logic in stored redactionLog, always set value in RedactionLogMerge

Merge branch 'RED-7918-processremoval' into 'release/3.222.x'
RED-7918 - RM-62: "Last Modified" date not updated after change See merge request redactmanager/redaction-service!205
2023-12-04 16:14:38 +01:00 · 2023-11-23 08:46:57 +01:00 · 2023-11-23 08:46:57 +01:00 · 2023-10-26 14:28:15 +02:00 · 2023-10-25 12:57:42 +02:00 · 2023-10-25 12:57:42 +02:00
41 changed files with 701 additions and 1865 deletions
@@ -0,0 +1,6 @@
+variables:
+  SONAR_PROJECT_KEY: 'RED_redaction-service'
+include:
+  - project: 'gitlab/gitlab'
+    ref: 'main'
+    file: 'ci-templates/maven_java.yml'
@@ -1,37 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>com.atlassian.bamboo</groupId>
-        <artifactId>bamboo-specs-parent</artifactId>
-        <version>8.1.3</version>
-        <relativePath/>
-    </parent>
-
-    <artifactId>bamboo-specs</artifactId>
-    <version>1.0.0-SNAPSHOT</version>
-    <packaging>jar</packaging>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.atlassian.bamboo</groupId>
-            <artifactId>bamboo-specs-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.atlassian.bamboo</groupId>
-            <artifactId>bamboo-specs</artifactId>
-        </dependency>
-
-        <!-- Test dependencies -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-
-    <!-- run 'mvn test' to perform offline validation of the plan -->
-    <!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
-</project>
@@ -1,156 +0,0 @@
-package buildjob;
-
-import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
-
-import java.time.LocalTime;
-
-import com.atlassian.bamboo.specs.api.BambooSpec;
-import com.atlassian.bamboo.specs.api.builders.BambooKey;
-import com.atlassian.bamboo.specs.api.builders.Variable;
-import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
-import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
-import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
-import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions;
-import com.atlassian.bamboo.specs.api.builders.plan.Job;
-import com.atlassian.bamboo.specs.api.builders.plan.Plan;
-import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
-import com.atlassian.bamboo.specs.api.builders.plan.Stage;
-import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
-import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
-import com.atlassian.bamboo.specs.api.builders.project.Project;
-import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
-import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
-import com.atlassian.bamboo.specs.builders.task.ScriptTask;
-import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
-import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
-import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
-import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
-import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
-import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
-import com.atlassian.bamboo.specs.util.BambooServer;
-
-/**
- * Plan configuration for Bamboo.
- * Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
- */
-@BambooSpec
-public class PlanSpec {
-
-    private static final String SERVICE_NAME = "redaction-service";
-
-    private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
-
-    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
-
-
-    /**
-     * Run main to publish plan on Bamboo
-     */
-    public static void main(final String[] args) {
-        //By default credentials are read from the '.credentials' file.
-        BambooServer bambooServer = new BambooServer("http://localhost:8085");
-
-        Plan plan = new PlanSpec().createPlan();
-        bambooServer.publish(plan);
-        PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
-        bambooServer.publish(planPermission);
-
-        Plan nightPlan = new PlanSpec().createNightPlan();
-        bambooServer.publish(nightPlan);
-        PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
-        bambooServer.publish(nightPlanPermission);
-
-        Plan secPlan = new PlanSpec().createSecBuild();
-        bambooServer.publish(secPlan);
-        PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
-        bambooServer.publish(secPlanPermission);
-    }
-
-
-    private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
-
-        Permissions permission = new Permissions().userPermissions("atlbamboo",
-                        PermissionType.EDIT,
-                        PermissionType.VIEW,
-                        PermissionType.ADMIN,
-                        PermissionType.CLONE,
-                        PermissionType.BUILD)
-                .groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
-                .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
-                .loggedInUserPermissions(PermissionType.VIEW)
-                .anonymousUserPermissionView();
-        return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
-    }
-
-
-    private Project project() {
-
-        return new Project().name("RED").key(new BambooKey("RED"));
-    }
-
-
-    public Plan createPlan() {
-
-        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
-                .variables(new Variable("maven_add_param", ""))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
-                                        .enabled(true),
-                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
-                                new ScriptTask().description("Build").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh").argument(SERVICE_NAME),
-                                createJUnitParserTask().description("Resultparser")
-                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                        .enabled(true),
-                                new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL),
-                                new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(new BitbucketServerTrigger())
-                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
-                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
-                        .notificationForCommitters());
-    }
-
-
-    public Plan createNightPlan() {
-
-        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
-                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
-                                        .enabled(true),
-                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
-                                new ScriptTask().description("Build")
-                                        .location(Location.FILE)
-                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
-                                        .argument(SERVICE_NAME + " verify"),
-                                createJUnitParserTask().description("Resultparser")
-                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
-                                        .enabled(true))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
-                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
-    }
-
-
-    public Plan createSecBuild() {
-
-        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
-                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
-                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
-                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
-                                new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                .dockerRunArguments("--net=host")
-                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
-                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
-                .linkedRepositories("RED / " + SERVICE_NAME)
-                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
-                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
-    }
-
-}
@@ -1,65 +0,0 @@
-#!/bin/bash
-set -e
-
-SERVICE_NAME=$1
-MVN_TARGET=${2:-deploy}
-
-if [[ "$bamboo_planRepository_branchName" == "master" ]]
-then
-    branchVersion=$(cat pom.xml | grep -Eo "    <version>.*-SNAPSHOT</version>" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|' | tr -d ' ')
-    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
-    newVersion="$(semver $latestVersion -p -i minor)"
-    echo "new release on master with version $newVersion"
-elif [[ "$bamboo_planRepository_branchName" == release* ]]
-then
-    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
-    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
-    newVersion="$(semver $latestVersion -p -i patch)"
-    echo "new release on $bamboo_planRepository_branchName with version $newVersion"
-elif [[ "${bamboo_version_tag}" != "dev" ]]
-then
-    newVersion="${bamboo_version_tag}"
-    echo "new special version bild with $newVersion"
-else
-    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        --no-transfer-progress \
-        ${bamboo_maven_add_param} \
-        clean install \
-        -Djava.security.egd=file:/dev/./urandomelse
-    echo "dev build with tag ${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
-    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
-    exit 0
-fi
-
-echo "gitTag=${newVersion}" > git.tag
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    ${bamboo_maven_add_param} \
-    versions:set \
-    -DnewVersion=${newVersion}
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    ${bamboo_maven_add_param} \
-    versions:set  \
-    -DnewVersion=${newVersion}
-
-mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    --no-transfer-progress \
-    clean $MVN_TARGET \
-    ${bamboo_maven_add_param} \
-    -e \
-    -DdeployAtEnd=true \
-    -Dmaven.wagon.http.ssl.insecure=true \
-    -Dmaven.wagon.http.ssl.allowall=true \
-    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    package
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    docker:push
@@ -1,44 +0,0 @@
-#!/bin/bash
-set -e
-
-SERVICE_NAME=$1
-
-echo "build jar binaries"
-mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    --no-transfer-progress \
-    clean install \
-    -Djava.security.egd=file:/dev/./urandomelse
-
-echo "dependency-check:aggregate"
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    org.owasp:dependency-check-maven:aggregate
-
-if [[ -z "${bamboo_repository_pr_key}" ]]
-then
-    echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
-    mvn --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        sonar:sonar \
-        -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
-        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-        -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-else
-    echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
-    mvn --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        sonar:sonar \
-        -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
-        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-        -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-        -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-        -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-fi
@@ -1,24 +0,0 @@
-package buildjob;
-
-import org.junit.Test;
-
-import com.atlassian.bamboo.specs.api.builders.plan.Plan;
-import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
-import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
-
-public class PlanSpecTest {
-
-    @Test
-    public void checkYourPlanOffline() throws PropertiesValidationException {
-
-        Plan plan = new PlanSpec().createPlan();
-        EntityPropertiesBuilders.build(plan);
-
-        Plan nightPlan = new PlanSpec().createNightPlan();
-        EntityPropertiesBuilders.build(nightPlan);
-
-        Plan secPlan = new PlanSpec().createSecBuild();
-        EntityPropertiesBuilders.build(secPlan);
-    }
-
-}
@@ -13,7 +13,6 @@
    <packaging>pom</packaging>

    <modules>
-        <module>bamboo-specs</module>
        <module>redaction-service-v1</module>
        <module>redaction-service-image-v1</module>
    </modules>
@@ -3,9 +3,9 @@
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
-        <groupId>com.iqser.red</groupId>
+        <groupId>com.knecon.fforesight</groupId>
        <artifactId>platform-docker-dependency</artifactId>
-        <version>1.2.0</version>
+        <version>0.1.0</version>
        <relativePath/>
    </parent>
    <modelVersion>4.0.0</modelVersion>
@@ -5,7 +5,7 @@
    <parent>
        <artifactId>platform-dependency</artifactId>
        <groupId>com.iqser.red</groupId>
-        <version>1.17.0</version>
+        <version>1.13.0</version>
        <relativePath/>
    </parent>
    <modelVersion>4.0.0</modelVersion>
@@ -30,6 +30,7 @@
        <dependency>
            <groupId>com.iqser.red.commons</groupId>
            <artifactId>storage-commons</artifactId>
+            <version>1.8.5</version>
        </dependency>

        <dependency>
@@ -1,5 +1,9 @@
 package com.iqser.red.service.redaction.v1.server;

+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -9,10 +13,6 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;

-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;

@@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;

+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@@ -25,13 +25,16 @@ public final class RulingTextDirAdjustUtil {
    private Point2D convertPoint(float x, float y, float dir, float pageWidth, float pageHeight) {

        var xAdj = getXRot(x, y, dir, pageWidth, pageHeight);
-        var yLowerLeftRot = getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
-        var yAdj = dir == 0 || dir == 180 ? pageHeight - yLowerLeftRot : pageWidth - yLowerLeftRot;
+        var yAdj = 0f;
+        if (dir == 0 || dir == 180) {
+            yAdj = pageHeight - getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
+        } else {
+            yAdj = pageWidth - getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
+        }
        return new Point2D.Float(xAdj, yAdj);
    }


-    @SuppressWarnings("SuspiciousNameCombination")
    private float getXRot(float x, float y, float dir, float pageWidth, float pageHeight) {

        if (dir == 0) {
@@ -225,6 +225,9 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public String getFont() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "none";
+        }
        return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
    }

@@ -233,6 +236,9 @@ public class TextPositionSequence implements CharSequence {
    @JsonAttribute(ignore = true)
    public String getFontStyle() {

+        if (textPositions.get(0).getFontName() == null) {
+            return "standard";
+        }
        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();

        if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
@@ -14,8 +14,8 @@ import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;

 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -1589,7 +1589,7 @@ public class Section {
        if (values != null) {
            for (String value : values) {
                if (StringUtils.isNotBlank(value)) {
-                    String[] lines = value.split("\n");
+                    String[] lines = value.trim().split("\n");

                    for (String line : lines) {

@@ -1,25 +1,33 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
+package com.iqser.red.service.redaction.v1.server.redaction.service;

 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;

 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;

 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;

+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
@@ -36,94 +44,42 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionChangeLogService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.SectionGridCreatorService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.SectionTextBuilderService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction.EntityRedactionService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;

 import io.micrometer.core.annotation.Timed;
-import io.micrometer.core.instrument.FunctionTimer;
-import io.micrometer.core.instrument.MeterRegistry;
-import lombok.AccessLevel;
-import lombok.Getter;
+import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@RequiredArgsConstructor
 public class AnalyzeService {

-    DictionaryService dictionaryService;
-    DroolsExecutionService droolsExecutionService;
-    EntityRedactionService entityRedactionService;
-    RedactionLogCreatorService redactionLogCreatorService;
-    RedactionStorageService redactionStorageService;
-    PdfSegmentationService pdfSegmentationService;
-    RedactionChangeLogService redactionChangeLogService;
-    LegalBasisClient legalBasisClient;
-    RedactionServiceSettings redactionServiceSettings;
-    SectionTextBuilderService sectionTextBuilderService;
-    SectionGridCreatorService sectionGridCreatorService;
-    ImageService imageService;
-    ImportedRedactionService importedRedactionService;
-    SectionFinder sectionFinder;
-    PagewiseCounter analyzeCounter = new PagewiseCounter();
-
-
-    @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
-    public AnalyzeService(DictionaryService dictionaryService,
-                          DroolsExecutionService droolsExecutionService,
-                          EntityRedactionService entityRedactionService,
-                          RedactionLogCreatorService redactionLogCreatorService,
-                          RedactionStorageService redactionStorageService,
-                          PdfSegmentationService pdfSegmentationService,
-                          RedactionChangeLogService redactionChangeLogService,
-                          LegalBasisClient legalBasisClient,
-                          RedactionServiceSettings redactionServiceSettings,
-                          SectionTextBuilderService sectionTextBuilderService,
-                          SectionGridCreatorService sectionGridCreatorService,
-                          ImageService imageService,
-                          ImportedRedactionService importedRedactionService,
-                          SectionFinder sectionFinder,
-                          MeterRegistry meterRegistry) {
-
-        this.dictionaryService = dictionaryService;
-        this.droolsExecutionService = droolsExecutionService;
-        this.entityRedactionService = entityRedactionService;
-        this.redactionLogCreatorService = redactionLogCreatorService;
-        this.redactionStorageService = redactionStorageService;
-        this.pdfSegmentationService = pdfSegmentationService;
-        this.redactionChangeLogService = redactionChangeLogService;
-        this.legalBasisClient = legalBasisClient;
-        this.redactionServiceSettings = redactionServiceSettings;
-        this.sectionTextBuilderService = sectionTextBuilderService;
-        this.sectionGridCreatorService = sectionGridCreatorService;
-        this.imageService = imageService;
-        this.importedRedactionService = importedRedactionService;
-        this.sectionFinder = sectionFinder;
-
-        // This is just here as an example of how to do a timer with a counter.
-        // For a full implementation this should be moved to a common helper class.
-        FunctionTimer.builder("redactmanager_analyze.pagewise", //
-                        analyzeCounter, //
-                        PagewiseCounter::getTotalPageCount, //
-                        PagewiseCounter::getTotalTimeMillis, //
-                        TimeUnit.MILLISECONDS) //
-                .register(meterRegistry);
-    }
+    private final DictionaryService dictionaryService;
+    private final DroolsExecutionService droolsExecutionService;
+    private final EntityRedactionService entityRedactionService;
+    private final RedactionLogCreatorService redactionLogCreatorService;
+    private final RedactionStorageService redactionStorageService;
+    private final PdfSegmentationService pdfSegmentationService;
+    private final RedactionChangeLogService redactionChangeLogService;
+    private final LegalBasisClient legalBasisClient;
+    private final RedactionServiceSettings redactionServiceSettings;
+    private final SectionTextBuilderService sectionTextBuilderService;
+    private final SectionGridCreatorService sectionGridCreatorService;
+    private final ImageService imageService;
+    private final ImportedRedactionService importedRedactionService;


    @Timed("redactmanager_analyzeDocumentStructure")
@@ -197,25 +153,32 @@ public class AnalyzeService {
                new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
                analyzeRequest.getDossierId());

-        Set<Integer> sectionsToReanalyse = analyzeRequest.getSectionsToReanalyse().isEmpty() //
-                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest) //
-                : analyzeRequest.getSectionsToReanalyse();
-        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
+        Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse().isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(
+                dictionaryIncrement,
+                redactionLog,
+                text,
+                analyzeRequest);

        if (sectionsToReanalyse.isEmpty()) {
            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
        }

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        } else {
+            nerEntities = new NerEntities();
+        }
+
        List<SectionText> reanalysisSections = text.getSectionTexts()
                .stream()
                .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
                .collect(Collectors.toList());
-        NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() //
-                ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) //
-                : new NerEntities();
+
        KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());

+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);

        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
@@ -275,13 +238,48 @@ public class AnalyzeService {
    }


+    @Timed("redactmanager_findSectionsToReanalyse")
+    private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
+
+        long start = System.currentTimeMillis();
+        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+
+        Set<Integer> sectionsToReanalyse = new HashSet<>();
+        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
+        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
+            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
+                sectionsToReanalyse.add(entry.getSectionNumber());
+            }
+            if (entry.isImage()) {
+                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
+            }
+        }
+
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
+                true);
+
+        for (SectionText sectionText : text.getSectionTexts()) {
+
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
+                sectionsToReanalyse.add(sectionText.getSectionNumber());
+            }
+
+        }
+
+        log.info("Should reanalyze {} sections for request: {}, took: {}", sectionsToReanalyse.size(), analyzeRequest, System.currentTimeMillis() - start);
+
+        return sectionsToReanalyse;
+    }
+
+
    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                           long startTime,
                                           RedactionLog redactionLog,
                                           Text text,
                                           DictionaryVersion dictionaryVersion,
                                           boolean isReanalysis,
-                                           Set<FileAttribute> addedFileAttributes) {
+                                           Set<FileAttribute> addedFileAttributes
+    ) {

        redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
        redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
@@ -296,8 +294,6 @@ public class AnalyzeService {

        long duration = System.currentTimeMillis() - startTime;

-        analyzeCounter.increase(text.getNumberOfPages(), duration);
-
        return AnalyzeResult.builder()
                .dossierId(analyzeRequest.getDossierId())
                .fileId(analyzeRequest.getFileId())
@@ -317,12 +313,41 @@ public class AnalyzeService {
    }


+    private Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null) {
+            return new HashSet<>();
+        }
+
+        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
+                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
+                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
+                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
+                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+    }
+
+
    public List<RedactionLogLegalBasis> convert(List<LegalBasis> legalBasis) {

        return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
    }


+    public Image convert(RedactionLogEntry entry) {
+
+        Rectangle position = entry.getPositions().get(0);
+
+        return Image.builder()
+                .type(entry.getType())
+                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
+                .sectionNumber(entry.getSectionNumber())
+                .section(entry.getSection())
+                .page(position.getPage())
+                .hasTransparency(entry.isImageHasTransparency())
+                .build();
+    }
+
+
    private void excludeExcludedPages(RedactionLog redactionLog, Set<Integer> excludedPages) {

        if (excludedPages != null && !excludedPages.isEmpty()) {
@@ -347,21 +372,4 @@ public class AnalyzeService {
        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
    }

-
-    @FieldDefaults(level = AccessLevel.PRIVATE)
-    @Getter
-    private static final class PagewiseCounter {
-
-        long totalPageCount;
-        long totalTimeMillis;
-
-
-        public void increase(long pageCount, long durationMillis) {
-
-            totalPageCount += pageCount;
-            totalTimeMillis += durationMillis;
-        }
-
-    }
-
 }
@@ -1,52 +1,41 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
+package com.iqser.red.service.redaction.v1.server.redaction.service;

 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

-import lombok.AccessLevel;
+import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
@Slf4j
@Service
@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityRedactionService {

-    DroolsExecutionService droolsExecutionService;
-    SurroundingWordsService surroundingWordsService;
-    EntityFinder entityFinder;
+    private final RedactionServiceSettings redactionServiceSettings;
+    private final DroolsExecutionService droolsExecutionService;
+    private final SurroundingWordsService surroundingWordsService;


    public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@@ -56,7 +45,7 @@ public class EntityRedactionService {

        if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {

-            if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
+            if(!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
                //AnalyzeRequest provides immutable list.
                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
                mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
@@ -65,14 +54,7 @@ public class EntityRedactionService {
            }

            Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
-            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
-                    dictionary,
-                    kieContainer,
-                    analyzeRequest,
-                    true,
-                    hintsPerSectionNumber,
-                    imagesPerPage,
-                    nerEntities);
+            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities);
            EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
            EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
        }
@@ -85,78 +67,18 @@ public class EntityRedactionService {


    public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
-                                           Dictionary dictionary,
-                                           KieContainer kieContainer,
-                                           AnalyzeRequest analyzeRequest,
-                                           boolean local,
-                                           Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                           Map<Integer, Set<Image>> imagesPerPage,
-                                           NerEntities nerEntities) {
+                                    Dictionary dictionary,
+                                    KieContainer kieContainer,
+                                    AnalyzeRequest analyzeRequest,
+                                    boolean local,
+                                    Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                    Map<Integer, Set<Image>> imagesPerPage,
+                                    NerEntities nerEntities) {

-        List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
-                dictionary,
-                analyzeRequest,
-                local,
-                hintsPerSectionNumber,
-                nerEntities);
+        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
+        for (SectionText reanalysisSection : reanalysisSections) {

-        Set<FileAttribute> addedFileAttributes = new HashSet<>();
-        Set<Entity> entities = new HashSet<>();
-        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
-
-            if (!addedFileAttributes.isEmpty()) {
-                //Section.Builder provides immutable list.
-                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
-                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
-                mergedFileAttributes.addAll(addedFileAttributes);
-                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
-            }
-
-            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
-
-            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
-
-            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
-
-            var entriesWithoutSurroundingText = analysedSection.getEntities()
-                    .stream()
-                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
-                    .collect(Collectors.toSet());
-
-            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                        sectionSearchableTextPair.getSearchableText(),
-                        dictionary,
-                        sectionSearchableTextPair.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
-            }
-
-            entities.addAll(analysedSection.getEntities());
-
-            if (!local) {
-                for (Image image : analysedSection.getImages()) {
-                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
-                }
-                addLocalValuesToDictionary(analysedSection, dictionary);
-            }
-
-        });
-
-        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
-    }
-
-
-    private List<SectionSearchableTextPair> extractSearchableTextPairs(List<SectionText> reanalysisSections,
-                                                                       Dictionary dictionary,
-                                                                       AnalyzeRequest analyzeRequest,
-                                                                       boolean local,
-                                                                       Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                                                       NerEntities nerEntities) {
-
-        return reanalysisSections.stream().map(reanalysisSection -> {
-
-            Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
+            Entities entities = findEntities(reanalysisSection.getSearchableText(),
                    reanalysisSection.getHeadline(),
                    reanalysisSection.getSectionNumber(),
                    dictionary,
@@ -214,36 +136,73 @@ public class EntityRedactionService {

            log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());

-            return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);
-        }).collect(Collectors.toList());
-    }
+            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
+                    .isLocal(false)
+                    .dictionaryTypes(dictionary.getTypes())
+                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
+                            .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
+                    .nerEntities(entities.getNerEntities())
+                    .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
+                    .searchText(reanalysisSection.getSearchableText().toString())
+                    .headline(reanalysisSection.getHeadline())
+                    .sectionNumber(reanalysisSection.getSectionNumber())
+                    .tabularData(reanalysisSection.getTabularData())
+                    .searchableText(reanalysisSection.getSearchableText())
+                    .dictionary(dictionary)
+                    .images(reanalysisSection.getImages())
+                    .sectionAreas(reanalysisSection.getSectionAreas())
+                    .fileAttributes(analyzeRequest.getFileAttributes())
+                    .manualRedactions(analyzeRequest.getManualRedactions())
+                    .isInTable(reanalysisSection.isTable())
+                    .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
+
+        }


-    private SectionSearchableTextPair toSectionSearchableTextPair(Dictionary dictionary,
-                                                                  AnalyzeRequest analyzeRequest,
-                                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                                                  SectionText reanalysisSection,
-                                                                  Entities entities) {
+        Set<FileAttribute> addedFileAttributes = new HashSet<>();
+        Set<Entity> entities = new HashSet<>();
+        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {

-        return new SectionSearchableTextPair(Section.builder()
-                .isLocal(false)
-                .dictionaryTypes(dictionary.getTypes())
-                .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities().stream(),
-                        hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                .nerEntities(entities.getNerEntities())
-                .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
-                .searchText(reanalysisSection.getSearchableText().toString())
-                .headline(reanalysisSection.getHeadline())
-                .sectionNumber(reanalysisSection.getSectionNumber())
-                .tabularData(reanalysisSection.getTabularData())
-                .searchableText(reanalysisSection.getSearchableText())
-                .dictionary(dictionary)
-                .images(reanalysisSection.getImages())
-                .sectionAreas(reanalysisSection.getSectionAreas())
-                .fileAttributes(analyzeRequest.getFileAttributes())
-                .manualRedactions(analyzeRequest.getManualRedactions())
-                .isInTable(reanalysisSection.isTable())
-                .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts());
+            if(!addedFileAttributes.isEmpty()) {
+                //Section.Builder provides immutable list.
+                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
+                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
+                mergedFileAttributes.addAll(addedFileAttributes);
+                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
+            }
+
+            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
+
+            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
+
+            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+
+            var entriesWithoutSurroundingText = analysedSection.getEntities()
+                    .stream()
+                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
+                    .collect(Collectors.toSet());
+
+            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                        sectionSearchableTextPair.getSearchableText(),
+                        dictionary,
+                        sectionSearchableTextPair.getCellStarts());
+            } else {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+            }
+
+            entities.addAll(analysedSection.getEntities());
+
+            if (!local) {
+                for (Image image : analysedSection.getImages()) {
+                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
+                }
+                addLocalValuesToDictionary(analysedSection, dictionary);
+            }
+
+        });
+
+        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
    }


@@ -285,7 +244,7 @@ public class EntityRedactionService {
    private Map<Integer, Set<Entity>> getHintsPerSection(Set<Entity> entities, Dictionary dictionary) {

        Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
-        entities.forEach(entity -> {
+        entities.stream().forEach(entity -> {
            if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
                hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity);
            }
@@ -310,4 +269,95 @@ public class EntityRedactionService {
        }));
    }

+
+    @Timed("redactmanager_findEntities")
+    private Entities findEntities(SearchableText searchableText,
+                                  String headline,
+                                  int sectionNumber,
+                                  Dictionary dictionary,
+                                  boolean local,
+                                  NerEntities nerEntities,
+                                  List<Integer> cellStarts,
+                                  ManualRedactions manualRedactions) {
+
+        Set<Entity> found = new HashSet<>();
+        String searchableString = searchableText.asString();
+
+        if (StringUtils.isEmpty(searchableString)) {
+            return new Entities(new HashSet<>(), new HashSet<>());
+        }
+
+        String lowercaseInputString = searchableString.toLowerCase();
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+
+            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation,
+                    model,
+                    new FindEntityDetails(model.getType(),
+                            headline,
+                            sectionNumber,
+                            !local,
+                            model.isDossierDictionary(),
+                            local ? Engine.RULE : Engine.DICTIONARY,
+                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+
+            var withPositions = EntitySearchUtils.clearAndFindPositions(entities, searchableText, dictionary, manualRedactions);
+
+            EntitySearchUtils.addOrAddEngine(found, withPositions.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()));
+        }
+
+        Set<Entity> nerFound = new HashSet<>();
+        if (!local) {
+            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
+        }
+
+        EntitySearchUtils.removeEntitiesContainedInLarger(found);
+        return new Entities(found, nerFound);
+    }
+
+
+    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
+
+        Set<Entity> entities = new HashSet<>();
+
+        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
+            nerEntities.getData().get(sectionNumber).forEach(res -> {
+                if (cellStarts == null || cellStarts.isEmpty()) {
+                    entities.add(new Entity(res.getValue(),
+                            res.getType(),
+                            res.getStartOffset(),
+                            res.getEndOffset(),
+                            headline,
+                            sectionNumber,
+                            false,
+                            false,
+                            Engine.NER,
+                            EntityType.RECOMMENDATION));
+                } else {
+                    boolean intersectsCellStart = false;
+                    for (Integer cellStart : cellStarts) {
+                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
+                            intersectsCellStart = true;
+                            break;
+                        }
+                    }
+                    if (!intersectsCellStart) {
+                        entities.add(new Entity(res.getValue(),
+                                res.getType(),
+                                res.getStartOffset(),
+                                res.getEndOffset(),
+                                headline,
+                                sectionNumber,
+                                false,
+                                false,
+                                Engine.NER,
+                                EntityType.RECOMMENDATION));
+                    }
+                }
+            });
+        }
+        return entities;
+    }
+
 }
@@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.Change;
 import com.iqser.red.service.redaction.v1.model.ChangeType;
+import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.ManualChange;
 import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
 import com.iqser.red.service.redaction.v1.model.Point;
@@ -108,7 +109,7 @@ public class RedactionLogMergeService {

        var skippedImportedRedactions = new HashSet<>();
        log.info("Merging Redaction log with manual redactions");
-        if (manualRedactions != null) {
+        if (manualRedactions != null) {  // This is never null! Could be removed

            var manualRedactionLogEntries = addManualAddEntries(sectionGrid,
                    manualRedactions.getEntriesToAdd(),
@@ -135,8 +136,12 @@ public class RedactionLogMergeService {
                    entry.getPositions().forEach(pos -> {
                        if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
                            entry.setExcluded(true);
+                        } else {
+                            entry.setExcluded(false); // This is a workaround, because in the analysis exlcuded is also set, but should not, because it triggers no reanalyis
                        }
                    });
+                } else {
+                    entry.setExcluded(false); // This is a workaround, because in the analysis exlcuded is also set, but should not, because it triggers no reanalyis
                }
            }

@@ -207,157 +212,30 @@ public class RedactionLogMergeService {

        manualRedactionWrappers.forEach(mrw -> {

-            if (mrw.getItem() instanceof ManualImageRecategorization) {
-                var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
-                String manualOverrideReason = null;
-                if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-
-                    redactionLogEntry.setType(imageRecategorization.getType());
-                    redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
-
-                    if (isHint(types, imageRecategorization.getType())) {
-                        redactionLogEntry.setRedacted(false);
-                        redactionLogEntry.setHint(true);
-                    } else {
-                        redactionLogEntry.setHint(false);
-                    }
-
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
-                } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(imageRecategorization)
-                                .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                                .withChange("type", imageRecategorization.getType()));
+            Object item = mrw.getItem();
+            if (item instanceof ManualImageRecategorization) {
+                var imageRecategorization = (ManualImageRecategorization) item;
+                processManualImageRecategorization(redactionLogEntry, types, colors, imageRecategorization);
            }

-            if (mrw.getItem() instanceof IdRemoval) {
-                var manualRemoval = (IdRemoval) mrw.getItem();
-                if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
-                    log.debug("Skipping merge for dictionary-modifying entry");
-                } else {
-                    String manualOverrideReason = null;
-                    if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
-                        redactionLogEntry.setRedacted(false);
-                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), true, types));
-                        redactionLogEntry.setHint(false);
-                    } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                        manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    }
-
-                    if (manualOverrideReason != null) {
-                        redactionLogEntry.setReason(manualOverrideReason);
-                    }
-                }
-
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(manualRemoval)
-                                .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
+            if (item instanceof IdRemoval) {
+                var manualRemoval = (IdRemoval) item;
+                processIdRemoval(redactionLogEntry, types, colors, manualRemoval);
            }

-            if (mrw.getItem() instanceof ManualForceRedaction) {
-                var manualForceRedact = (ManualForceRedaction) mrw.getItem();
-                String manualOverrideReason = null;
-                var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
-                if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    // Forcing a skipped hint should result in a hint
-                    if (dictionaryIsHint) {
-                        redactionLogEntry.setHint(true);
-                    } else {
-                        redactionLogEntry.setRedacted(true);
-                    }
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                var manualChange = ManualChange.from(manualForceRedact)
-                        .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
-
-                redactionLogEntry.getManualChanges().add(manualChange);
-
+            if (item instanceof ManualForceRedaction) {
+                var manualForceRedact = (ManualForceRedaction) item;
+                processManualForceRedaction(redactionLogEntry, types, colors, manualForceRedact);
            }

-            if (mrw.getItem() instanceof ManualLegalBasisChange) {
-                var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
-                String manualOverrideReason = null;
-                if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
-                    redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-                    redactionLogEntry.setRedacted(true);
-                    if (manualLegalBasisChange.getSection() != null) {
-                        redactionLogEntry.setSection(manualLegalBasisChange.getSection());
-                    }
-                    if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                        redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-                    }
-                } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                }
-
-                if (manualOverrideReason != null) {
-                    redactionLogEntry.setReason(manualOverrideReason);
-                }
-
-                var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
-                manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
-                if (manualLegalBasisChange.getSection() != null) {
-                    manualChange.withChange("section", manualLegalBasisChange.getSection());
-                }
-                if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                    manualChange.withChange("value", manualLegalBasisChange.getValue());
-                }
-                redactionLogEntry.getManualChanges().add(manualChange);
+            if (item instanceof ManualLegalBasisChange) {
+                var manualLegalBasisChange = (ManualLegalBasisChange) item;
+                processManualLegalBasisChange(redactionLogEntry, types, colors, manualLegalBasisChange);
            }

-            if (mrw.getItem() instanceof ManualResizeRedaction) {
-                var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
-                String manualOverrideReason = null;
-                if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-                    if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
-                        redactionLogEntry.setValue(manualResizeRedact.getValue());
-                    }
-                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                    }
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
-                } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                    manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-                    redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-
-                    // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-                    if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                        redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                        redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-                    }
-                }
-
-                redactionLogEntry.setReason(manualOverrideReason);
-                redactionLogEntry.getManualChanges()
-                        .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
+            if (item instanceof ManualResizeRedaction) {
+                var manualResizeRedact = (ManualResizeRedaction) item;
+                processManualResizeRedaction(redactionLogEntry, types, colors, manualResizeRedact);
            }

        });
@@ -365,6 +243,36 @@ public class RedactionLogMergeService {
    }


+    private void processManualImageRecategorization(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualImageRecategorization imageRecategorization) {
+
+        String manualOverrideReason = null;
+        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
+
+            redactionLogEntry.setType(imageRecategorization.getType());
+            redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
+
+            if (isHint(types, imageRecategorization.getType())) {
+                redactionLogEntry.setRedacted(false);
+                redactionLogEntry.setHint(true);
+            } else {
+                redactionLogEntry.setHint(false);
+            }
+
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
+        } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
+            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
+        }
+
+        if (manualOverrideReason != null) {
+            redactionLogEntry.setReason(manualOverrideReason);
+        }
+
+        redactionLogEntry.getManualChanges()
+                .add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
+    }
+
+
    private String mergeReasonIfNecessary(String currentReason, String addition) {

        if (currentReason != null) {
@@ -378,6 +286,138 @@ public class RedactionLogMergeService {
    }


+    private void processIdRemoval(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, IdRemoval manualRemoval) {
+
+        boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
+        if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && manualRemoval.getProcessedDate() != null && isBasedOnDictionaryOnly(redactionLogEntry)) {
+            log.debug("Skipping merge for dictionary-modifying entry");
+        } else {
+            String redactionLogEntryType = redactionLogEntry.getType();
+            String manualOverrideReason = null;
+            if (isApprovedRedaction) {
+                redactionLogEntry.setRedacted(false);
+                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
+                redactionLogEntry.setColor(getColor(redactionLogEntryType, colors, false, redactionLogEntry.isRedacted(), true, types));
+                redactionLogEntry.setHint(false);
+            } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
+                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
+                redactionLogEntry.setColor(getColor(redactionLogEntryType, colors, true, redactionLogEntry.isRedacted(), false, types));
+            }
+
+            if (manualOverrideReason != null) {
+                redactionLogEntry.setReason(manualOverrideReason);
+            }
+        }
+
+        redactionLogEntry.getManualChanges()
+                .add(ManualChange.from(manualRemoval)
+                        .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
+    }
+
+
+    private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
+
+        return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
+    }
+
+
+    private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualForceRedaction manualForceRedact) {
+
+        String manualOverrideReason = null;
+        var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
+        if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
+            // Forcing a skipped hint should result in a hint
+            if (dictionaryIsHint) {
+                redactionLogEntry.setHint(true);
+            } else {
+                redactionLogEntry.setRedacted(true);
+            }
+            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
+            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+        } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
+            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
+            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
+        }
+
+        if (manualOverrideReason != null) {
+            redactionLogEntry.setReason(manualOverrideReason);
+        }
+
+        var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
+
+        redactionLogEntry.getManualChanges().add(manualChange);
+    }
+
+
+    private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualLegalBasisChange manualLegalBasisChange) {
+
+        String manualOverrideReason = null;
+        if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
+            redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
+            redactionLogEntry.setRedacted(true);
+            if (manualLegalBasisChange.getSection() != null) {
+                redactionLogEntry.setSection(manualLegalBasisChange.getSection());
+            }
+            if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
+                redactionLogEntry.setValue(manualLegalBasisChange.getValue());
+            }
+        } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
+            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
+        }
+
+        if (manualOverrideReason != null) {
+            redactionLogEntry.setReason(manualOverrideReason);
+        }
+
+        var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
+        manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
+        if (manualLegalBasisChange.getSection() != null) {
+            manualChange.withChange("section", manualLegalBasisChange.getSection());
+        }
+        if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
+            manualChange.withChange("value", manualLegalBasisChange.getValue());
+        }
+        redactionLogEntry.getManualChanges().add(manualChange);
+    }
+
+
+    private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualResizeRedaction manualResizeRedact) {
+
+        String manualOverrideReason = null;
+        if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
+            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
+            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
+            if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
+                redactionLogEntry.setValue(manualResizeRedact.getValue());
+            }
+            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
+            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
+                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
+                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
+            }
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
+        } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
+            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
+            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
+            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
+
+            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
+            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
+                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
+                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
+            }
+        }
+
+        redactionLogEntry.setReason(manualOverrideReason);
+        redactionLogEntry.getManualChanges()
+                .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
+    }
+
+
    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
                                                       Set<ManualRedactionEntry> manualAdds,
                                                       Map<String, List<Comment>> comments,
@@ -1,101 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Component;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.classification.model.Text;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Component
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-class SectionFinder {
-    @Timed("redactmanager_findSectionsToReanalyse")
-    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
-
-        long start = System.currentTimeMillis();
-        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
-
-        Set<Integer> sectionsToReanalyse = new HashSet<>();
-        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
-        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                sectionsToReanalyse.add(entry.getSectionNumber());
-            }
-            if (entry.isImage()) {
-                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
-            }
-        }
-
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
-
-        for (SectionText sectionText : text.getSectionTexts()) {
-
-            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
-                sectionsToReanalyse.add(sectionText.getSectionNumber());
-            }
-
-        }
-
-        log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
-
-        return sectionsToReanalyse;
-    }
-
-    private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
-
-        if (manualRedactions == null) {
-            return new HashSet<>();
-        }
-
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
-    }
-
-    private static Image convert(RedactionLogEntry entry) {
-
-        Rectangle position = entry.getPositions().get(0);
-
-        return Image.builder()
-                .type(entry.getType())
-                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
-                .sectionNumber(entry.getSectionNumber())
-                .section(entry.getSection())
-                .page(position.getPage())
-                .hasTransparency(entry.isImageHasTransparency())
-                .build();
-    }
-}
@@ -1,127 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.stereotype.Component;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Component
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-class EntityFinder {
-
-    RedactionServiceSettings redactionServiceSettings;
-
-
-    @Timed("redactmanager_findEntities")
-    public Entities findEntities(SearchableText searchableText,
-                                 String headline,
-                                 int sectionNumber,
-                                 Dictionary dictionary,
-                                 boolean local,
-                                 NerEntities nerEntities,
-                                 List<Integer> cellStarts,
-                                 ManualRedactions manualRedactions) {
-
-        Set<Entity> found = new HashSet<>();
-        String searchableString = searchableText.asString();
-
-        if (StringUtils.isEmpty(searchableString)) {
-            return new Entities(new HashSet<>(), new HashSet<>());
-        }
-
-        String lowercaseInputString = searchableString.toLowerCase();
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
-
-            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
-            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
-                    searchImplementation,
-                    model,
-                    new FindEntityDetails(model.getType(),
-                            headline,
-                            sectionNumber,
-                            !local,
-                            model.isDossierDictionary(),
-                            local ? Engine.RULE : Engine.DICTIONARY,
-                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
-
-            EntitySearchUtils.addOrAddEngine(found, entities);
-        }
-
-        Set<Entity> nerFound = new HashSet<>();
-        if (!local) {
-            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
-        }
-
-        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
-        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
-    }
-
-
-    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
-
-        Set<Entity> entities = new HashSet<>();
-
-        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
-            nerEntities.getData().get(sectionNumber).forEach(res -> {
-                if (cellStarts == null || cellStarts.isEmpty()) {
-                    entities.add(new Entity(res.getValue(),
-                            res.getType(),
-                            res.getStartOffset(),
-                            res.getEndOffset(),
-                            headline,
-                            sectionNumber,
-                            false,
-                            false,
-                            Engine.NER,
-                            EntityType.RECOMMENDATION));
-                } else {
-                    boolean intersectsCellStart = false;
-                    for (Integer cellStart : cellStarts) {
-                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
-                            intersectsCellStart = true;
-                            break;
-                        }
-                    }
-                    if (!intersectsCellStart) {
-                        entities.add(new Entity(res.getValue(),
-                                res.getType(),
-                                res.getStartOffset(),
-                                res.getEndOffset(),
-                                headline,
-                                sectionNumber,
-                                false,
-                                false,
-                                Engine.NER,
-                                EntityType.RECOMMENDATION));
-                    }
-                }
-            });
-        }
-        return entities;
-    }
-
-}
@@ -273,14 +273,14 @@ public final class EntitySearchUtils {
                existing.setLegalBasis(found.getLegalBasis());
                existing.setMatchedRule(found.getMatchedRule());
                existing.setRedactionReason(found.getRedactionReason());
-                if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType()
-                        .equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
+                if (isOneARecommendationAndTheOtherEntity(found, existing)) {
                    existing.setEntityType(EntityType.ENTITY);
                    if (found.isRedaction()) {
                        existing.setRedaction(true);
                    }
                }
-            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
+            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType()) &&
+                    !(isOneARecommendationAndTheOtherEntity(found, existing) && existing.isRedaction() && found.isRedaction()) ) {
                entities.remove(found);
                entities.add(found);
            }
@@ -289,6 +289,13 @@ public final class EntitySearchUtils {
        }
    }

+    private boolean isOneARecommendationAndTheOtherEntity(Entity entityOne, Entity entityTwo) {
+
+        var entityTypeOne = entityOne.getEntityType();
+        var entityTypeTwo = entityTwo.getEntityType();
+        return entityTypeTwo.equals(EntityType.RECOMMENDATION) && entityTypeOne.equals(EntityType.ENTITY)
+                || entityTypeTwo.equals(EntityType.ENTITY) && entityTypeOne.equals(EntityType.RECOMMENDATION);
+    }

    public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found) {
        // HashSet keeps old value but we want the new.
@@ -53,14 +53,15 @@ public class PdfSegmentationService {
    public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<PdfImage>> pdfImages) throws IOException {

        PDDocument pdDocument = null;
-        File tempFile = null;
        try {
+            //create tempFile
+            File tempFile = FileUtils.createTempFile("document", ".pdf");
+
            Map<Integer, List<PdfTableCell>> pdfTableCells = new HashMap<>();
            if (redactionServiceSettings.isCvTableParsingEnabled()) {
                pdfTableCells = tableService.convertTables(dossierId, fileId);
            }

-            tempFile = FileUtils.createTempFile("document", ".pdf");
            try (var fos = new FileOutputStream(tempFile)) {
                IOUtils.copy(documentInputStream, fos);

@@ -73,7 +74,44 @@ public class PdfSegmentationService {
                long pageCount = pdDocument.getNumberOfPages();

                for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-                    processPage(pdfImages, pdDocument, pdfTableCells, document, pages, pageNumber);
+
+                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+                    stripper.setPageNumber(pageNumber);
+                    stripper.setStartPage(pageNumber);
+                    stripper.setEndPage(pageNumber);
+                    stripper.setPdpage(pdPage);
+                    stripper.getText(pdDocument);
+
+                    PDRectangle pdr = pdPage.getMediaBox();
+
+                    int rotation = pdPage.getRotation();
+                    boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+                    PDRectangle cropbox = pdPage.getCropBox();
+                    CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                            stripper.getRulings(),
+                            stripper.getMinCharWidth(),
+                            stripper.getMaxCharHeight());
+                    Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+
+                    page.setRotation(rotation);
+                    page.setLandscape(isLandscape);
+                    page.setPageNumber(pageNumber);
+                    page.setPageWidth(cropbox.getWidth());
+                    page.setPageHeight(cropbox.getHeight());
+
+                    // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+                    if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+                        page.setImages(pdfImages.get(pageNumber));
+                        imageService.findOcr(page);
+                    }
+
+                    tableExtractionService.extractTables(cleanRulings, page);
+                    buildPageStatistics(page);
+                    increaseDocumentStatistics(page, document);
+
+                    pages.add(page);
                }

                document.setPages(pages);
@@ -82,65 +120,22 @@ public class PdfSegmentationService {
                sectionsBuilderService.buildSections(document);
                sectionsBuilderService.addImagesToSections(document);

+                IOUtils.close(pdDocument);
+
+                if (!tempFile.delete()) {
+                    log.warn("Could not delete tmp file");
+                }
+
                return document;
            }
        } finally {
            if (pdDocument != null) {
                pdDocument.close();
            }
-
-            FileUtils.deleteFile(tempFile);
        }
    }


-    private void processPage(Map<Integer, List<PdfImage>> pdfImages,
-                           PDDocument pdDocument,
-                           Map<Integer, List<PdfTableCell>> pdfTableCells,
-                           Document document,
-                           List<Page> pages,
-                           int pageNumber) throws IOException {
-
-        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-        stripper.setPageNumber(pageNumber);
-        stripper.setStartPage(pageNumber);
-        stripper.setEndPage(pageNumber);
-        stripper.setPdpage(pdPage);
-        stripper.getText(pdDocument);
-
-        PDRectangle pdr = pdPage.getMediaBox();
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-        PDRectangle cropbox = pdPage.getCropBox();
-        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                stripper.getRulings(),
-                stripper.getMinCharWidth(),
-                stripper.getMaxCharHeight());
-        Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-        page.setRotation(rotation);
-        page.setLandscape(isLandscape);
-        page.setPageNumber(pageNumber);
-        page.setPageWidth(cropbox.getWidth());
-        page.setPageHeight(cropbox.getHeight());
-
-        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-            page.setImages(pdfImages.get(pageNumber));
-            imageService.findOcr(page);
-        }
-
-        tableExtractionService.extractTables(cleanRulings, page);
-        buildPageStatistics(page);
-        increaseDocumentStatistics(page, document);
-
-        pages.add(page);
-    }
-
-
    private void increaseDocumentStatistics(Page page, Document document) {

        if (!page.isLandscape()) {
@@ -25,6 +25,8 @@ public class Cell extends Rectangle {

    private boolean isHeaderCell;

+    private static final int MIN_SIZE = 1;
+

    public Cell(Point2D topLeft, Point2D bottomRight) {

@@ -66,4 +68,10 @@ public class Cell extends Rectangle {
        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
    }

+
+    public boolean hasMinimumSize() {
+
+        return this.getHeight() >= MIN_SIZE && this.getWidth() >= MIN_SIZE;
+    }
+
 }
@@ -261,7 +261,9 @@ public class Table extends AbstractTextContainer {
                    if (intersectionCell.isPresent()) {
                        cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
                    }
-                    row.add(cell);
+                    if (cell.hasMinimumSize()) {
+                        row.add(cell);
+                    }
                }
                prevX = x;
            }
@@ -88,7 +88,7 @@ public class TableExtractionService {
        for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
            TextBlock textBlock = (TextBlock) abstractTextContainer;
            for (Cell cell : cells) {
-                if (cell.intersects(textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
                        textBlock.getPdfMinY(),
                        textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                        textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -109,7 +109,7 @@ public class TableExtractionService {

            List<Cell> overlappingCells = new ArrayList<>();
            for (Cell c : cells) {
-                if (c.intersects(area)) {
+                if (c.hasMinimumSize() && c.intersects(area)) {
                    overlappingCells.add(c);
                }
            }
@@ -20,31 +20,19 @@ public class FileUtils {
    }


-    /**
-     * Deletes a file; logs a message with the reason if the deletion fails.
-     * This method is null-safe.
-     * @param file The file to delete.  Can be null.
-     */
-    public void deleteFile(File file) {
-        if (file != null) {
-            try {
-                Files.deleteIfExists(file.toPath());
-            } catch (IOException ex) {
-                log.warn("Could not delete file!", ex);
-            }
-        }
-    }
-
-
    // We don't need to check the results of the permission setters below,
    // since we're manipulating a file we created ourselves.
-    @SuppressWarnings({"ResultOfMethodCallIgnored", "squid:S899"})
+    @SuppressWarnings("ResultOfMethodCallIgnored")
    private void setRWPermissionsOnlyForOwner(File tempFile) {

        try {
+            // deny for all
+            tempFile.setReadable(false);
+            tempFile.setWritable(false);
+            tempFile.setExecutable(false);
+            // allow for owner
            tempFile.setReadable(true, true);
            tempFile.setWritable(true, true);
-            tempFile.setExecutable(false);
        } catch (SecurityException ex) {
            // This should never happen since we're creating a temp file ourselves.
            log.warn("Caught an exception during temp file creation.  This should not happend.  Check the code.", ex);
@@ -1,503 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static org.mockito.Mockito.when;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.stubbing.Answer;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-@ExtendWith(SpringExtension.class)
-@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(AbstractTestWithDictionaries.TestConfiguration.class)
-public class AbstractTestWithDictionaries {
-
-    protected static final String RULES = loadFromClassPath("drools/rules.drl");
-    protected static final String RULES_PATH = "drools/rules.drl";
-    protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
-    private static final String VERTEBRATE = "vertebrate";
-    private static final String ADDRESS = "CBI_address";
-    private static final String AUTHOR = "CBI_author";
-    private static final String SPONSOR = "CBI_sponsor";
-    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
-    private static final String REDACTION_INDICATOR = "redaction_indicator";
-    private static final String HINT_ONLY = "hint_only";
-    private static final String MUST_REDACT = "must_redact";
-    private static final String PUBLISHED_INFORMATION = "published_information";
-    private static final String TEST_METHOD = "test_method";
-    private static final String PURITY = "purity";
-    private static final String IMAGE = "image";
-    private static final String LOGO = "logo";
-    private static final String SIGNATURE = "signature";
-    private static final String FORMULA = "formula";
-    private static final String OCR = "ocr";
-    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
-    private static final String IMPORTED_REDACTION = "imported_redaction";
-    private static final String PII = "PII";
-    private static final String ROTATE_SIMPLE = "RotateSimple";
-
-    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
-    protected final static String TEST_DOSSIER_ID = "123";
-    protected final static String TEST_FILE_ID = "123";
-
-    @Autowired
-    protected StorageService storageService;
-
-    @MockBean
-    protected DictionaryClient dictionaryClient;
-
-    @MockBean
-    protected RulesClient rulesClient;
-
-    private final Map<String, List<String>> dictionary = new HashMap<>();
-    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
-    private final Map<String, List<String>> falsePositive = new HashMap<>();
-    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
-    private final Map<String, String> typeColorMap = new HashMap<>();
-    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
-    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
-    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
-    private final Map<String, Integer> rankTypeMap = new HashMap<>();
-    private final Colors colors = new Colors();
-    private final Map<String, Long> reanalysisVersions = new HashMap<>();
-    private final Set<String> deleted = new HashSet<>();
-
-
-    @BeforeEach
-    public void stubClients() {
-
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
-
-        loadDictionaryForTest();
-        loadTypeForTest();
-        loadNerForTest();
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
-
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
-                .build()));
-
-        mockDictionaryCalls(null);
-        mockDictionaryCalls(0L);
-
-        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
-    }
-
-
-    private static String loadFromClassPath(String path) {
-
-        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
-        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-            StringBuilder sb = new StringBuilder();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
-        }
-    }
-
-
-    private void mockDictionaryCalls(Long version) {
-
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
-                false));
-        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                NO_REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
-                false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                PUBLISHED_INFORMATION,
-                false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
-                false));
-        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
-        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
-        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
-        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
-        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
-        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
-                false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                DOSSIER_REDACTIONS,
-                true));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                IMPORTED_REDACTION,
-                true));
-
-    }
-
-
-    private String cleanDictionaryEntry(String entry) {
-
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-
-
-    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
-
-        if (entries == null) {
-            entries = Collections.emptyList();
-        }
-
-        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
-        entries.forEach(entry -> {
-            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
-        });
-        return dictionaryEntries;
-    }
-
-
-    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
-
-        return Type.builder()
-                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .hexColor(typeColorMap.get(type))
-                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
-                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
-                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
-                .isHint(hintTypeMap.get(type))
-                .isCaseInsensitive(caseInSensitiveMap.get(type))
-                .isRecommendation(recommendationTypeMap.get(type))
-                .rank(rankTypeMap.get(type))
-                .build();
-    }
-
-
-    private void loadTypeForTest() {
-
-        typeColorMap.put(VERTEBRATE, "#ff85f7");
-        typeColorMap.put(ADDRESS, "#ffe187");
-        typeColorMap.put(AUTHOR, "#ffe187");
-        typeColorMap.put(SPONSOR, "#85ebff");
-        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
-        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
-        typeColorMap.put(HINT_ONLY, "#abc0c4");
-        typeColorMap.put(MUST_REDACT, "#fab4c0");
-        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
-        typeColorMap.put(TEST_METHOD, "#91fae8");
-        typeColorMap.put(PII, "#66ccff");
-        typeColorMap.put(PURITY, "#ffe187");
-        typeColorMap.put(IMAGE, "#fcc5fb");
-        typeColorMap.put(OCR, "#fcc5fb");
-        typeColorMap.put(LOGO, "#ffe187");
-        typeColorMap.put(FORMULA, "#ffe187");
-        typeColorMap.put(SIGNATURE, "#ffe187");
-        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
-        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
-
-        hintTypeMap.put(VERTEBRATE, true);
-        hintTypeMap.put(ADDRESS, false);
-        hintTypeMap.put(AUTHOR, false);
-        hintTypeMap.put(SPONSOR, false);
-        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
-        hintTypeMap.put(REDACTION_INDICATOR, true);
-        hintTypeMap.put(HINT_ONLY, true);
-        hintTypeMap.put(MUST_REDACT, true);
-        hintTypeMap.put(PUBLISHED_INFORMATION, true);
-        hintTypeMap.put(TEST_METHOD, true);
-        hintTypeMap.put(PII, false);
-        hintTypeMap.put(PURITY, false);
-        hintTypeMap.put(IMAGE, true);
-        hintTypeMap.put(OCR, true);
-        hintTypeMap.put(FORMULA, false);
-        hintTypeMap.put(LOGO, false);
-        hintTypeMap.put(SIGNATURE, false);
-        hintTypeMap.put(DOSSIER_REDACTIONS, false);
-        hintTypeMap.put(IMPORTED_REDACTION, false);
-        hintTypeMap.put(ROTATE_SIMPLE, false);
-
-        caseInSensitiveMap.put(VERTEBRATE, true);
-        caseInSensitiveMap.put(ADDRESS, false);
-        caseInSensitiveMap.put(AUTHOR, false);
-        caseInSensitiveMap.put(SPONSOR, false);
-        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(HINT_ONLY, true);
-        caseInSensitiveMap.put(MUST_REDACT, true);
-        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
-        caseInSensitiveMap.put(TEST_METHOD, false);
-        caseInSensitiveMap.put(PII, false);
-        caseInSensitiveMap.put(PURITY, false);
-        caseInSensitiveMap.put(IMAGE, true);
-        caseInSensitiveMap.put(OCR, true);
-        caseInSensitiveMap.put(SIGNATURE, true);
-        caseInSensitiveMap.put(LOGO, true);
-        caseInSensitiveMap.put(FORMULA, true);
-        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
-        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
-        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
-
-        recommendationTypeMap.put(VERTEBRATE, false);
-        recommendationTypeMap.put(ADDRESS, false);
-        recommendationTypeMap.put(AUTHOR, false);
-        recommendationTypeMap.put(SPONSOR, false);
-        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(HINT_ONLY, false);
-        recommendationTypeMap.put(MUST_REDACT, false);
-        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
-        recommendationTypeMap.put(TEST_METHOD, false);
-        recommendationTypeMap.put(PII, false);
-        recommendationTypeMap.put(PURITY, false);
-        recommendationTypeMap.put(IMAGE, false);
-        recommendationTypeMap.put(OCR, false);
-        recommendationTypeMap.put(FORMULA, false);
-        recommendationTypeMap.put(SIGNATURE, false);
-        recommendationTypeMap.put(LOGO, false);
-        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
-        recommendationTypeMap.put(IMPORTED_REDACTION, false);
-        recommendationTypeMap.put(ROTATE_SIMPLE, false);
-
-        rankTypeMap.put(PURITY, 155);
-        rankTypeMap.put(PII, 150);
-        rankTypeMap.put(ADDRESS, 140);
-        rankTypeMap.put(AUTHOR, 130);
-        rankTypeMap.put(SPONSOR, 120);
-        rankTypeMap.put(VERTEBRATE, 110);
-        rankTypeMap.put(MUST_REDACT, 100);
-        rankTypeMap.put(REDACTION_INDICATOR, 90);
-        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
-        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
-        rankTypeMap.put(TEST_METHOD, 60);
-        rankTypeMap.put(HINT_ONLY, 50);
-        rankTypeMap.put(IMAGE, 30);
-        rankTypeMap.put(OCR, 29);
-        rankTypeMap.put(LOGO, 28);
-        rankTypeMap.put(SIGNATURE, 27);
-        rankTypeMap.put(FORMULA, 26);
-        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
-        rankTypeMap.put(IMPORTED_REDACTION, 200);
-        rankTypeMap.put(ROTATE_SIMPLE, 150);
-
-        colors.setSkippedColor("#cccccc");
-        colors.setRequestAddColor("#04b093");
-        colors.setRequestRemoveColor("#04b093");
-    }
-
-
-    @SneakyThrows
-    private void loadNerForTest() {
-
-        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
-    }
-
-
-    private List<Type> getTypeResponse() {
-
-        return typeColorMap.entrySet()
-                .stream()
-                .map(typeColor -> Type.builder()
-                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                        .type(typeColor.getKey())
-                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                        .hexColor(typeColor.getValue())
-                        .isHint(hintTypeMap.get(typeColor.getKey()))
-                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
-                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
-                        .rank(rankTypeMap.get(typeColor.getKey()))
-                        .build())
-
-                .collect(Collectors.toList());
-    }
-
-
-    protected List<File> getPathsRecursively(File path) {
-
-        List<File> result = new ArrayList<>();
-        if (path == null || path.listFiles() == null) {
-            return result;
-        }
-        for (File f : path.listFiles()) {
-            if (f.isFile()) {
-                result.add(f);
-            } else {
-                result.addAll(getPathsRecursively(f));
-            }
-        }
-        return result;
-
-    }
-
-
-    protected void loadOnlyDictionaryForSimpleFile() {
-
-        dictionary.clear();
-        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-    }
-
-
-    private void loadDictionaryForTest() {
-
-        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
-
-        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(String file) {
-
-        return prepareStorage(file, "files/cv_service_empty_response.json");
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
-
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
-        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
-
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
-    }
-
-
-    @SneakyThrows
-    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
-
-        AnalyzeRequest request = AnalyzeRequest.builder()
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .lastProcessed(OffsetDateTime.now())
-                .build();
-
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
-
-        return request;
-    }
-
-
-    @AfterEach
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
-        }
-    }
-
-
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
-    public static class TestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService();
-        }
-
-    }
-
-}
@@ -1,213 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import javax.servlet.ServletContext;
-
-import org.apache.commons.collections4.SetUtils;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.core.io.ClassPathResource;
-
-import com.google.common.io.Files;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import lombok.Builder;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class AnalysisIntegrationTest extends AbstractTestWithDictionaries {
-
-    @Autowired
-    private AnalyzeService analyzeService;
-
-    @Autowired
-    private ServletContext context;
-
-    @Autowired
-    private RedactionStorageService redactionStorageService;
-
-    @MockBean
-    private RabbitTemplate rabbitTemplate;
-
-    @MockBean
-    private LegalBasisClient legalBasisClient;
-
-
-    @Test
-    public void testAnalysisIntegrationForAllFiles() {
-
-        findAllFilesWithRedactionLogs().forEach((pdfFile, redactionLogFile) -> {
-
-            try {
-                String relativePdfFilePath = new ClassPathResource("").getURI().relativize(pdfFile.toURI()).getPath();
-                log.info("Checking equality for {}", relativePdfFilePath);
-                AnalyzeRequest request = prepareStorage(relativePdfFilePath);
-
-                analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
-                analyzeService.analyze(request);
-
-                RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-                Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-
-                Path entityReferencePath = Paths.get(new ClassPathResource("entities/").getFile().getAbsolutePath().replace("target/test-classes/", "src/test/resources/"),
-                        relativePdfFilePath.replace("files/", "").replace(".pdf", ".json"));
-
-                new File(entityReferencePath.toString().replace(entityReferencePath.getFileName().toString(), "")).mkdirs();
-
-                try (var out = new FileOutputStream(entityReferencePath.toString())) {
-                    ObjectMapperFactory.create().writeValue(out, references);
-                    log.info("{} Entities have been written for File {}", references.size(), relativePdfFilePath);
-                }
-                //RedactionLog originalRedactionLog = ObjectMapperFactory.create().readValue(redactionLogFile, RedactionLog.class);
-                //assertEntitiesAreEqual(originalRedactionLog, redactionLog);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-
-        });
-    }
-
-
-    // only use this if you must replace all entities with newer versions
-    @Test
-    @Disabled
-    @SneakyThrows
-    public void writeNewEntitiesForAllFiles() {
-
-        ClassPathResource fileStem = new ClassPathResource("files/");
-        List<File> allPdfFiles = getAllPdfFilesAsStream(fileStem).toList();
-        System.out.printf("found %d PDF Files to create Entities for\n", allPdfFiles.size());
-        int count = 0;
-        for (File pdfFile : allPdfFiles) {
-            writeNewEntities(pdfFile);
-            System.out.printf("%d|%d finished writing entities for %s\n", count, allPdfFiles.size(), pdfFile.getName());
-            count++;
-        }
-    }
-
-
-    private static Stream<File> getAllPdfFilesAsStream(ClassPathResource fileStem) throws IOException {
-
-        return Arrays.stream(fileStem.getFile().listFiles())
-                .filter(File::isDirectory)
-                .flatMap(file -> Arrays.stream(file.listFiles()))
-                .filter(file -> Files.getFileExtension(file.getName()).equals("pdf"))
-                .filter(File::isFile)
-                .filter(File::canRead);
-    }
-
-
-    @SneakyThrows
-    private void writeNewEntities(File pdfFile) {
-
-        Path relativePdfFilePath = Paths.get(pdfFile.toString().replace(new ClassPathResource("").getFile().getAbsolutePath(), ""));
-        Path relativeFolderPath = relativePdfFilePath.getParent();
-        Path absoluteEntityFolderPath = Paths.get(context.getContextPath(), "src", "test", "resources", "entities", relativeFolderPath.toString().replace("files/", ""));
-        absoluteEntityFolderPath.toFile().mkdirs();
-        Path absoluteEntityFilePath = absoluteEntityFolderPath.resolve(relativePdfFilePath.getFileName().toString().replace("files/", "").replace(".pdf", ".json"));
-
-        AnalyzeRequest request = prepareStorage(relativePdfFilePath.toString());
-
-        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
-        analyzeService.analyze(request);
-
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-
-        try (var out = new FileOutputStream(absoluteEntityFilePath.toString())) {
-            ObjectMapperFactory.create().writeValue(out, references);
-        }
-
-    }
-
-
-    private void assertEntitiesAreEqual(RedactionLog originalRedactionLog, RedactionLog redactionLog) {
-
-        Set<EntityReference> originalReferences = originalRedactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-        Set<EntityReference> references = redactionLog.getRedactionLogEntry().stream().map(this::toReference).collect(Collectors.toSet());
-
-        Set<EntityReference> missingReferences = SetUtils.difference(originalReferences, references);
-        Set<EntityReference> additionalReferences = SetUtils.difference(references, originalReferences);
-
-        if (missingReferences.size() == 0) {
-            log.info("All original Entities have been found!");
-        } else {
-            log.warn("{} original References not found {}", missingReferences.size(), missingReferences);
-            assert missingReferences.size() == 0;
-        }
-
-        if (additionalReferences.size() == 0) {
-            log.info("No additional Entities have been found!");
-        } else {
-            log.warn("{} additional Entities found", additionalReferences.size());
-            additionalReferences.forEach(e -> log.warn(e.toString()));
-            assert additionalReferences.size() == 0;
-        }
-    }
-
-
-    private EntityReference toReference(RedactionLogEntry redactionLogEntry) {
-
-        return EntityReference.builder()
-                .positions(redactionLogEntry.getPositions())
-                .isRecommendation(redactionLogEntry.isRecommendation())
-                .isFalsePositive(redactionLogEntry.isFalsePositive())
-                .word(redactionLogEntry.getValue())
-                .isHint(redactionLogEntry.isHint())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .reason(redactionLogEntry.getReason())
-                .build();
-    }
-
-
-    @Builder
-    public record EntityReference(
-            String word, String reason, boolean redacted, boolean isHint, boolean isRecommendation, boolean isFalsePositive, boolean isDictionaryEntry, List<Rectangle> positions) {
-
-    }
-
-
-    @SneakyThrows
-    private Map<File, File> findAllFilesWithRedactionLogs() {
-
-        ClassPathResource fileStem = new ClassPathResource("files/");
-        ClassPathResource redactionStem = new ClassPathResource("RedactionLog/files/");
-
-        Map<File, File> fileToRedactionLogFile = new HashMap<>();
-        List<File> allFiles = getAllPdfFilesAsStream(fileStem).toList();
-
-        for (File file : allFiles) {
-            String relativePath = fileStem.getURI().relativize(file.toURI()).getPath().replace(".pdf", ".json");
-            File redactionLogFile = Paths.get(redactionStem.getFile().getAbsolutePath(), relativePath).toFile();
-            if (redactionLogFile.isFile() && redactionLogFile.canRead()) {
-                fileToRedactionLogFile.put(file, redactionLogFile);
-            }
-        }
-        return fileToRedactionLogFile;
-    }
-
-}
@@ -1,28 +1,5 @@
 package com.iqser.red.service.redaction.v1.server;

-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.when;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
@@ -32,7 +9,28 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.DictionarySer
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;

-@ExtendWith(SpringExtension.class)
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;
+
+@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class DictionaryServiceTest {
@@ -20,10 +20,10 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import org.assertj.core.api.Assertions;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -41,7 +41,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.context.junit4.SpringRunner;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -59,8 +59,8 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -72,7 +72,7 @@ import lombok.EqualsAndHashCode;
 import lombok.SneakyThrows;
 import lombok.ToString;

-@ExtendWith(SpringExtension.class)
+@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(HeadlinesGoldStandardIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class HeadlinesGoldStandardIntegrationTest {
@@ -229,7 +229,7 @@ public class HeadlinesGoldStandardIntegrationTest {
    }


-    @AfterEach
+    @After
    public void cleanupStorage() {

        if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -238,7 +238,7 @@ public class HeadlinesGoldStandardIntegrationTest {
    }


-    @BeforeEach
+    @Before
    public void stubClients() {

        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
@@ -24,11 +24,11 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;

-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -46,7 +46,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.context.junit4.SpringRunner;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -81,8 +81,8 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
@@ -92,7 +92,7 @@ import com.iqser.red.storage.commons.service.StorageService;

 import lombok.SneakyThrows;

-@ExtendWith(SpringExtension.class)
+@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class RedactionIntegrationTest {
@@ -202,7 +202,7 @@ public class RedactionIntegrationTest {
    }


-    @AfterEach
+    @After
    public void cleanupStorage() {

        if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -211,7 +211,7 @@ public class RedactionIntegrationTest {
    }


-    @BeforeEach
+    @Before
    public void stubClients() {

        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
@@ -293,7 +293,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Disabled
+    @Ignore
    public void testLargeScannedFileOOM() {

        AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
@@ -398,7 +398,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Disabled
+    @Ignore
    @SneakyThrows
    public void testIgnoreHint() {

@@ -440,7 +440,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Disabled
+    @Ignore
    public void noExceptionShouldBeThrownForAnyFiles() throws IOException {

        long start = System.currentTimeMillis();
@@ -1252,7 +1252,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Disabled
+    @Ignore
    public void resizeRedactionTest() throws IOException {

        String pdfFile = "files/Minimal Examples/Single Table.pdf";
@@ -1369,7 +1369,7 @@ public class RedactionIntegrationTest {


    @Test
-    @Disabled
+    @Ignore
    public void testManualSurroundingText() throws IOException {

        String pdfFile = "files/new/S4.pdf";
@@ -28,12 +28,11 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import org.apache.commons.lang3.StringUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -50,7 +49,8 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.annotation.IfProfileValue;
+import org.springframework.test.context.junit4.SpringRunner;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -76,8 +76,8 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@@ -88,7 +88,7 @@ import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;

@Slf4j
-@ExtendWith(SpringExtension.class)
+@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
@@ -247,7 +247,7 @@ public class RulesTest {
    private LegalBasisClient legalBasisClient;


-    @BeforeEach
+    @Before
    public void stubClients() {

        objectMapper.registerModule(new JavaTimeModule());
@@ -280,7 +280,7 @@ public class RulesTest {
    }


-    @AfterEach
+    @After
    public void cleanupStorage() {

        if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -294,7 +294,7 @@ public class RulesTest {
     * If the RedactionLog already exists, it will be overwritten
     * Test is ignored, because it's for manual tests.
     */
-    @Disabled
+    @Ignore
    @Test
    public void generateRedactionLogForOneFile() {

@@ -336,7 +336,7 @@ public class RulesTest {
     * Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
     * If RedactionLog Json for one file does not exist, whole test will fail.
     */
-    @EnabledIfSystemProperty(named = "test-groups", matches = "(.*)rules-test(.*)")
+    @IfProfileValue(name = "test-groups", value = "rules-test")
    @Test
    public void analyseAllFilesAndCompareRedactionLogs() {

@@ -4,7 +4,7 @@ import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Set;

-import org.junit.jupiter.api.Test;
+import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;

@@ -1,21 +1,28 @@
 package com.iqser.red.service.redaction.v1.server.realdata;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.nullable;
-import static org.mockito.Mockito.when;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.*;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;

-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
+import io.micrometer.prometheus.PrometheusMeterRegistry;
+import lombok.SneakyThrows;

 import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -28,32 +35,18 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.context.junit4.SpringRunner;

-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
-import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;

-import io.micrometer.prometheus.PrometheusMeterRegistry;
-import lombok.SneakyThrows;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.when;

-@ExtendWith(SpringExtension.class)
+@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class LiveDataIntegrationTest {
@@ -115,7 +108,7 @@ public class LiveDataIntegrationTest {


    @SneakyThrows
-    @BeforeEach
+    @Before
    public void prepareTest() {

        when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
@@ -1,11 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.redaction.rulebuilder;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-
-import org.junit.jupiter.api.Test;
-
 import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;

+import org.junit.Test;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
 public class RuleBuilderModelServiceTest {

    @Test
@@ -5,7 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import java.util.HashSet;
 import java.util.Set;

-import org.junit.jupiter.api.Test;
+import org.junit.Test;

 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
@@ -6,7 +6,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 import org.apache.commons.lang3.StringUtils;
-import org.junit.jupiter.api.Test;
+import org.junit.Test;

 public class RegExPatternTest {

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;

 import org.assertj.core.api.Assertions;
-import org.junit.jupiter.api.Test;
+import org.junit.Test;

 public class TextNormalizationUtilitiesTest {

@@ -11,8 +11,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;

-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,7 +25,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.context.junit4.SpringRunner;

 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -49,7 +49,7 @@ import com.iqser.red.storage.commons.service.StorageService;

 import lombok.SneakyThrows;

-@ExtendWith(SpringExtension.class)
+@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(PdfSegmentationServiceTest.TestConfiguration.class)
 public class PdfSegmentationServiceTest {
@@ -1,24 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.stringmatching;

-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit4.SpringRunner;

 import java.util.HashSet;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;

-import org.ahocorasick.trie.Trie;
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;

-import lombok.AllArgsConstructor;
-import lombok.EqualsAndHashCode;
-import lombok.SneakyThrows;
-
-@ExtendWith(SpringExtension.class)
+@RunWith(SpringRunner.class)
 public class StringMatchingPerformanceTest {

    @Test
@@ -34,7 +34,7 @@ public class StringMatchingPerformanceTest {
        System.out.println("Loaded text has a length of " + text.length() + " symbols");
        System.out.println("Dictionary has " + dictionary.size() + " entries");

-        var patterns = dictionary.stream().map(p -> Pattern.compile(Pattern.quote(p))).toList();
+        var patterns = dictionary.stream().map(p -> Pattern.compile(Pattern.quote(p))).collect(Collectors.toList());
        var trie = Trie.builder().ignoreCase().addKeywords(dictionary).build();

        // 1. Naive approach
@@ -56,7 +56,7 @@ public class StringMatchingPerformanceTest {
        t1 = System.currentTimeMillis();
        var boyerMooreIndexes = new HashSet<Index>();
        for (var pattern : patterns) {
-            boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).toList());
+            boyerMooreIndexes.addAll(pattern.matcher(text).results().map(r -> new Index(r.start(), r.end())).collect(Collectors.toList()));
        }
        t2 = System.currentTimeMillis();
        System.out.println("Boyer Moore found " + boyerMooreIndexes.size() + " entries in " + (t2 - t1) + "ms");
Author	SHA1	Message	Date
Dominique Eifländer	d574ef0866	RM-76: Ignore excluded page logic in stored redactionLog, always set value in RedactionLogMerge	2023-12-04 16:14:38 +01:00
Corina Olariu	c76b79de2d	Merge branch 'RED-7918-processremoval' into 'release/3.222.x' RED-7918 - RM-62: "Last Modified" date not updated after change See merge request redactmanager/redaction-service!205	2023-11-23 08:46:57 +01:00
Corina Olariu	787ef98b76	RED-7918 - RM-62: "Last Modified" date not updated after change	2023-11-23 08:46:57 +01:00
Dominique Eifländer	f3b1345531	hotfix: readded fallback if files are stored uncompressed	2023-10-26 14:28:15 +02:00
Corina Olariu	ccf5e40f7f	Merge branch 'RED-7806-backport' into 'release/3.222.x' RED-7806 - Specific customer document cannot be processed See merge request redactmanager/redaction-service!175	2023-10-25 12:57:42 +02:00
Corina Olariu	669ae9d667	RED-7806 - Specific customer document cannot be processed	2023-10-25 12:57:42 +02:00
Dominique Eifländer	d62b9ac04d	Merge branch 'RED-7508-3.6' into 'release/3.222.x' RED-7508: Fixed removal of wrong false positive entries See merge request redactmanager/redaction-service!112	2023-08-29 13:51:31 +02:00
deiflaender	230179fe46	RED-7508: Fixed removal of wrong false positive entries	2023-08-29 13:45:09 +02:00
Kresnadi Budisantoso	f1fb0e6237	Merge branch 'hotfix/3.222.x-rule29' into 'release/3.222.x' RED-7040 Syngenta rule 29 does not work See merge request redactmanager/redaction-service!17	2023-06-29 12:02:46 +02:00
Kresnadi Budisantoso	8ee33d82a7	RED-7040 Syngenta rule 29 does not work	2023-06-29 12:02:46 +02:00
Thomas Beyer	ada5df0048	Pull request #535 : RED-6619 - change iqser to knecon Merge in RED/redaction-service from RED-6619_3.222_1 to release/3.222.x * commit '7fcc6fac03a8b2c27e479ce90893e56a3a43b6b5': RED-6619 - change iqser to knecon	2023-05-04 10:41:43 +02:00
Thomas Beyer	7fcc6fac03	RED-6619 - change iqser to knecon	2023-05-04 10:23:21 +02:00
Thomas Beyer	d345643499	Pull request #533 : RED-6619 - backmerge Merge in RED/redaction-service from RED-6619_3.222_1 to release/3.222.x * commit 'e5b8f7f6bebebe878e15a7345fc2054058e42fc2': RED-6619 - backmerge	2023-05-04 09:56:03 +02:00
Thomas Beyer	e5b8f7f6be	RED-6619 - backmerge	2023-04-28 16:52:30 +02:00
Thomas Beyer	bdcd3c7add	Pull request #529 : RED-6411 3.222.0 1 Merge in RED/redaction-service from RED-6411_3.222.0_1 to release/3.222.x * commit 'c7643da4788b7c9fb97428c409ff6e3eb2e19919': RED-6411 - optimize method RED-6411 - extend logic when replacing entities with higher rank RED-6411 - remove false positives before calling EntitySearchUtils.addOrAddEngine()	2023-03-28 12:21:01 +02:00
Thomas Beyer	c7643da478	RED-6411 - optimize method	2023-03-28 11:23:52 +02:00
Viktor Seifert	fe8f20ba0c	Pull request #527 : RED-6476: Changed handling of manual-remove-redactions. Merge in RED/redaction-service from RED-6476 to release/3.x * commit 'c462b389fa975d130ebc9836feb39e410e6a7a24': RED-6476: Changed handling of manual-remove-redactions.	2023-03-27 15:37:46 +02:00
Thomas Beyer	8409b61f40	RED-6411 - extend logic when replacing entities with higher rank	2023-03-27 12:48:19 +02:00
Viktor Seifert	c462b389fa	RED-6476: Changed handling of manual-remove-redactions. For cases where a manual-remove-redaction was applied to a redaction, that has multiple source (e.g. dictionary + rule), we still apply it when merging the redaction-log, instead of skipping it. This prevents a redaction from reappearing, when it has been removed by the user.	2023-03-24 17:25:53 +01:00
Thomas Beyer	ddd4c42da7	RED-6411 - remove false positives before calling EntitySearchUtils.addOrAddEngine()	2023-03-24 13:26:04 +01:00