RED-6224: Made dictionary cache multitenancy ready

Merge in RED/redaction-service from RED-6250 to master

* commit 'fe5121b1edf784d5cc57abd9cd8e27f70dea0f4a':
  RED-6250: Switched to new metrics-commons version to not have to instantiate a factory (reduces code duplication)
  RED-6250: Changed function-timer metric to use code from metric commons

.gitlab-ci.yml

@@ -1,6 +0,0 @@
-variables:
-  SONAR_PROJECT_KEY: 'RED_redaction-service'
-include:
-  - project: 'gitlab/gitlab'
-    ref: 'main'
-    file: 'ci-templates/maven_java.yml'

bamboo-specs/pom.xml

@@ -0,0 +1,37 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.atlassian.bamboo</groupId>
+        <artifactId>bamboo-specs-parent</artifactId>
+        <version>8.1.3</version>
+        <relativePath/>
+    </parent>
+
+    <artifactId>bamboo-specs</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.atlassian.bamboo</groupId>
+            <artifactId>bamboo-specs-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.atlassian.bamboo</groupId>
+            <artifactId>bamboo-specs</artifactId>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+
+    <!-- run 'mvn test' to perform offline validation of the plan -->
+    <!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
+</project>

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -0,0 +1,156 @@
+package buildjob;
+
+import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
+
+import java.time.LocalTime;
+
+import com.atlassian.bamboo.specs.api.BambooSpec;
+import com.atlassian.bamboo.specs.api.builders.BambooKey;
+import com.atlassian.bamboo.specs.api.builders.Variable;
+import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
+import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
+import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
+import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions;
+import com.atlassian.bamboo.specs.api.builders.plan.Job;
+import com.atlassian.bamboo.specs.api.builders.plan.Plan;
+import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
+import com.atlassian.bamboo.specs.api.builders.plan.Stage;
+import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
+import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
+import com.atlassian.bamboo.specs.api.builders.project.Project;
+import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
+import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
+import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
+import com.atlassian.bamboo.specs.builders.task.ScriptTask;
+import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
+import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
+import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
+import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
+import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
+import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
+import com.atlassian.bamboo.specs.util.BambooServer;
+
+/**
+ * Plan configuration for Bamboo.
+ * Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
+ */
+@BambooSpec
+public class PlanSpec {
+
+    private static final String SERVICE_NAME = "redaction-service";
+
+    private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
+
+    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
+
+
+    /**
+     * Run main to publish plan on Bamboo
+     */
+    public static void main(final String[] args) {
+        //By default credentials are read from the '.credentials' file.
+        BambooServer bambooServer = new BambooServer("http://localhost:8085");
+
+        Plan plan = new PlanSpec().createPlan();
+        bambooServer.publish(plan);
+        PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
+        bambooServer.publish(planPermission);
+
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        bambooServer.publish(nightPlan);
+        PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
+        bambooServer.publish(nightPlanPermission);
+
+        Plan secPlan = new PlanSpec().createSecBuild();
+        bambooServer.publish(secPlan);
+        PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
+        bambooServer.publish(secPlanPermission);
+    }
+
+
+    private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
+
+        Permissions permission = new Permissions().userPermissions("atlbamboo",
+                        PermissionType.EDIT,
+                        PermissionType.VIEW,
+                        PermissionType.ADMIN,
+                        PermissionType.CLONE,
+                        PermissionType.BUILD)
+                .groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
+                .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
+                .loggedInUserPermissions(PermissionType.VIEW)
+                .anonymousUserPermissionView();
+        return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
+    }
+
+
+    private Project project() {
+
+        return new Project().name("RED").key(new BambooKey("RED"));
+    }
+
+
+    public Plan createPlan() {
+
+        return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
+                .variables(new Variable("maven_add_param", ""))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
+                                        .enabled(true),
+                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
+                                new ScriptTask().description("Build").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh").argument(SERVICE_NAME),
+                                createJUnitParserTask().description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true),
+                                new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL),
+                                new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new BitbucketServerTrigger())
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranch()
+                        .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
+                        .notificationForCommitters());
+    }
+
+
+    public Plan createNightPlan() {
+
+        return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
+                .variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
+                                        .enabled(true),
+                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
+                                new ScriptTask().description("Build")
+                                        .location(Location.FILE)
+                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
+                                        .argument(SERVICE_NAME + " verify"),
+                                createJUnitParserTask().description("Resultparser")
+                                        .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
+                                        .enabled(true))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
+    }
+
+
+    public Plan createSecBuild() {
+
+        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
+                .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
+                                        .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
+                                new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
+                                new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME))
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                                .dockerRunArguments("--net=host")
+                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
+                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                .linkedRepositories("RED / " + SERVICE_NAME)
+                .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
+                .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
+    }
+
+}

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+set -e
+
+SERVICE_NAME=$1
+MVN_TARGET=${2:-deploy}
+
+if [[ "$bamboo_planRepository_branchName" == "master" ]]
+then
+    branchVersion=$(cat pom.xml | grep -Eo "    <version>.*-SNAPSHOT</version>" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|' | tr -d ' ')
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
+    newVersion="$(semver $latestVersion -p -i minor)"
+    echo "new release on master with version $newVersion"
+elif [[ "$bamboo_planRepository_branchName" == release* ]]
+then
+    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
+    newVersion="$(semver $latestVersion -p -i patch)"
+    echo "new release on $bamboo_planRepository_branchName with version $newVersion"
+elif [[ "${bamboo_version_tag}" != "dev" ]]
+then
+    newVersion="${bamboo_version_tag}"
+    echo "new special version bild with $newVersion"
+else
+    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        --no-transfer-progress \
+        ${bamboo_maven_add_param} \
+        clean install \
+        -Djava.security.egd=file:/dev/./urandomelse
+    echo "dev build with tag ${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
+    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
+    exit 0
+fi
+
+echo "gitTag=${newVersion}" > git.tag
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    ${bamboo_maven_add_param} \
+    versions:set \
+    -DnewVersion=${newVersion}
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    ${bamboo_maven_add_param} \
+    versions:set  \
+    -DnewVersion=${newVersion}
+
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    --no-transfer-progress \
+    clean $MVN_TARGET \
+    ${bamboo_maven_add_param} \
+    -e \
+    -DdeployAtEnd=true \
+    -Dmaven.wagon.http.ssl.insecure=true \
+    -Dmaven.wagon.http.ssl.allowall=true \
+    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
+    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    package
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    docker:push

bamboo-specs/src/main/resources/scripts/sonar-java.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+
+SERVICE_NAME=$1
+
+echo "build jar binaries"
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    --no-transfer-progress \
+    clean install \
+    -Djava.security.egd=file:/dev/./urandomelse
+
+echo "dependency-check:aggregate"
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    org.owasp:dependency-check-maven:aggregate \
+    -DknownExploitedEnabled=false
+
+if [[ -z "${bamboo_repository_pr_key}" ]]
+then
+    echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
+else
+    echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
+        -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
+        -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
+fi

bamboo-specs/src/test/java/buildjob/PlanSpecTest.java

@@ -0,0 +1,24 @@
+package buildjob;
+
+import org.junit.Test;
+
+import com.atlassian.bamboo.specs.api.builders.plan.Plan;
+import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
+import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
+
+public class PlanSpecTest {
+
+    @Test
+    public void checkYourPlanOffline() throws PropertiesValidationException {
+
+        Plan plan = new PlanSpec().createPlan();
+        EntityPropertiesBuilders.build(plan);
+
+        Plan nightPlan = new PlanSpec().createNightPlan();
+        EntityPropertiesBuilders.build(nightPlan);
+
+        Plan secPlan = new PlanSpec().createSecBuild();
+        EntityPropertiesBuilders.build(secPlan);
+    }
+
+}

pom.xml

@@ -7,12 +7,13 @@
 
     <artifactId>redaction-service</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>3.0-SNAPSHOT</version>
+    <version>4.0-SNAPSHOT</version>
 
 
     <packaging>pom</packaging>
 
     <modules>
+        <module>bamboo-specs</module>
         <module>redaction-service-v1</module>
         <module>redaction-service-image-v1</module>
     </modules>

redaction-service-image-v1/pom.xml

@@ -3,9 +3,9 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <groupId>com.knecon.fforesight</groupId>
+        <groupId>com.iqser.red</groupId>
         <artifactId>platform-docker-dependency</artifactId>
-        <version>0.1.0</version>
+        <version>1.2.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>

redaction-service-v1/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.13.0</version>
+        <version>1.17.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>redaction-service-v1</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>1.0-SNAPSHOT</version>
+    <version>4.0-SNAPSHOT</version>
 
     <packaging>pom</packaging>
 
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>1.22.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -6,13 +6,13 @@
     <parent>
         <artifactId>redaction-service-v1</artifactId>
         <groupId>com.iqser.red.service</groupId>
-        <version>1.0-SNAPSHOT</version>
+        <version>4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.299.0</persistence-service.version>
+        <persistence-service.version>2.1.0</persistence-service.version>
     </properties>
 
     <dependencies>
@@ -32,13 +32,18 @@
 
         <dependency>
             <groupId>com.iqser.red.service</groupId>
-            <artifactId>persistence-service-api-v1</artifactId>
+            <artifactId>persistence-service-internal-api-v1</artifactId>
             <version>${persistence-service.version}</version>
             <exclusions>
+
                 <exclusion>
                     <groupId>com.iqser.red.service</groupId>
                     <artifactId>redaction-service-api-v1</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>persistence-service-api-v1</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java

@@ -1,40 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeRequest {
-
-    private MessageType messageType;
-
-    private String dossierId;
-    private String fileId;
-    private String dossierTemplateId;
-    private ManualRedactions manualRedactions;
-    private OffsetDateTime lastProcessed;
-    private int analysisNumber;
-
-    @Builder.Default
-    private Set<Integer> excludedPages = new HashSet<>();
-    @Builder.Default
-    private Set<Integer> sectionsToReanalyse = new HashSet<>();
-
-    @Builder.Default
-    private List<FileAttribute> fileAttributes = new ArrayList<>();
-
-}
-

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java

@@ -1,40 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.Set;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeResult {
-
-    private MessageType messageType;
-
-    private String dossierId;
-    private String fileId;
-    private long duration;
-    private int numberOfPages;
-    private boolean hasUpdates;
-    private long dictionaryVersion;
-    private long dossierDictionaryVersion;
-    private long rulesVersion;
-    private long legalBasisVersion;
-
-    private boolean wasReanalyzed;
-
-    private int analysisVersion;
-    private int analysisNumber;
-
-    private ManualRedactions manualRedactions;
-
-    private Set<FileAttribute> addedFileAttributes;
-
-}
-

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/CellRectangle.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class CellRectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java

@@ -1,20 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class Change {
-
-    private int analysisNumber;
-    private ChangeType type;
-    private OffsetDateTime dateTime;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ChangeType.java

@@ -1,7 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum ChangeType {
-    ADDED,
-    REMOVED,
-    CHANGED
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Engine.java

@@ -1,7 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum Engine {
-    DICTIONARY,
-    NER,
-    RULE
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/FileAttribute.java

@@ -1,19 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class FileAttribute {
-
-    private String id;
-    private String label;
-    private String placeholder;
-    private String value;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedaction.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImportedRedaction {
-
-    private String id;
-
-    @Builder.Default
-    private List<Rectangle> positions = new ArrayList<>();
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ImportedRedactions.java

@@ -1,24 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@Data
-@Builder
-@CompiledJson
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImportedRedactions {
-
-    @Builder.Default
-    private Map<Integer, List<ImportedRedaction>> importedRedactions = new HashMap<>();
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java

@@ -1,59 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-import java.util.HashMap;
-import java.util.Map;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Builder
-public class ManualChange {
-
-    private AnnotationStatus annotationStatus;
-    private ManualRedactionType manualRedactionType;
-    private OffsetDateTime processedDate;
-    private OffsetDateTime requestedDate;
-    private String userId;
-    private Map<String, String> propertyChanges = new HashMap<>();
-
-
-    public static ManualChange from(BaseAnnotation baseAnnotation) {
-
-        ManualChange manualChange = new ManualChange();
-        manualChange.annotationStatus = baseAnnotation.getStatus();
-        manualChange.processedDate = baseAnnotation.getProcessedDate();
-        manualChange.requestedDate = baseAnnotation.getRequestDate();
-        manualChange.userId = baseAnnotation.getUser();
-        return manualChange;
-    }
-
-
-    public boolean isProcessed() {
-
-        return processedDate != null;
-    }
-
-
-    public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        this.manualRedactionType = manualRedactionType;
-        return this;
-    }
-
-
-    public ManualChange withChange(String property, String value) {
-
-        this.propertyChanges.put(property, value);
-        return this;
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java

@@ -1,13 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum ManualRedactionType {
-    ADD_LOCALLY,
-    ADD_TO_DICTIONARY,
-    REMOVE_LOCALLY,
-    REMOVE_FROM_DICTIONARY,
-    FORCE_REDACT,
-    FORCE_HINT,
-    RECATEGORIZE,
-    LEGAL_BASIS_CHANGE,
-    RESIZE
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MessageType.java

@@ -1,10 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public enum MessageType {
-
-    ANALYSE,
-    REANALYSE,
-    STRUCTURE_ANALYSE,
-    SURROUNDING_TEXT
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java

@@ -1,15 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class Point {
-
-    private float x;
-    private float y;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ReanalyzeResult.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class ReanalyzeResult {
-
-    private RedactionLog redactionLog;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class Rectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-
-    private int page;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java

@@ -1,37 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@Data
-@CompiledJson
-@AllArgsConstructor
-@NoArgsConstructor
-public class RedactionLog {
-
-    /**
-     * Version 0 Redaction Logs have manual redactions merged inside them
-     * Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime.
-     */
-    private long analysisVersion;
-
-    /**
-     * Which analysis created this redactionLog.
-     */
-    private int analysisNumber;
-
-    private List<RedactionLogEntry> redactionLogEntry = new ArrayList<>();
-    private List<RedactionLogLegalBasis> legalBasis = new ArrayList<>();
-
-    private long dictionaryVersion = -1;
-    private long dossierDictionaryVersion = -1;
-    private long rulesVersion = -1;
-    private long legalBasisVersion = -1;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogChanges.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class RedactionLogChanges {
-
-    private RedactionLog redactionLog;
-    private boolean hasChanges;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogComment.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.OffsetDateTime;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionLogComment {
-
-    private long id;
-    private String user;
-    private String text;
-    private String annotationId;
-    private String fileId;
-    private OffsetDateTime date;
-    private OffsetDateTime softDeletedTime;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -1,96 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-
-import lombok.*;
-
-import java.util.*;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@EqualsAndHashCode
-public class RedactionLogEntry {
-
-    private String id;
-    private String type;
-    private String value;
-    private String reason;
-    private int matchedRule;
-    private boolean rectangle;
-    private String legalBasis;
-
-    private boolean imported;
-
-    private boolean redacted;
-    private boolean isHint;
-    private boolean isRecommendation;
-    private boolean isFalsePositive;
-
-    private String section;
-    private float[] color;
-
-    @Builder.Default
-    private List<Rectangle> positions = new ArrayList<>();
-    private int sectionNumber;
-
-    private String textBefore;
-    private String textAfter;
-
-    @Builder.Default
-    private List<RedactionLogComment> comments = new ArrayList<>();
-
-    private int startOffset;
-    private int endOffset;
-
-    private boolean isImage;
-    private boolean imageHasTransparency;
-
-    private boolean isDictionaryEntry;
-    private boolean isDossierDictionaryEntry;
-
-    private boolean excluded;
-
-    private String sourceId;
-
-    @EqualsAndHashCode.Exclude
-    @Builder.Default
-    private List<Change> changes = new ArrayList<>();
-
-    @EqualsAndHashCode.Exclude
-    @Builder.Default
-    private List<ManualChange> manualChanges = new ArrayList<>();
-
-    private Set<Engine> engines = new HashSet<>();
-
-    private Set<String> reference = new HashSet<>();
-
-    @Builder.Default
-    private Set<String> importedRedactionIntersections = new HashSet<>();
-
-
-    public boolean lastChangeIsRemoved() {
-
-        return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
-    }
-
-
-    public boolean isLocalManualRedaction() {
-
-        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
-    }
-
-
-    public boolean isManuallyRemoved() {
-
-        return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY && mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
-    }
-
-
-    private <T> Optional<T> last(List<T> list) {
-
-        return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogLegalBasis.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionLogLegalBasis {
-
-    private String name;
-    private String description;
-    private String reason;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java

@@ -1,34 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionRequest {
-
-    private String dossierId;
-    private String fileId;
-    private String dossierTemplateId;
-    private ManualRedactions manualRedactions;
-    @Builder.Default
-    private Set<Integer> excludedPages = new HashSet<>();
-
-    private Colors colors;
-    private List<Type> types;
-
-    private boolean includeFalsePositives;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class RedactionResult {
-
-    private byte[] document;
-    private int numberOfPages;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java

@@ -1,34 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class SectionArea {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-    private int page;
-    private String header;
-
-
-    public boolean contains(Rectangle other) {
-
-        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft()
-                .getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft()
-                .getY() + other.getHeight();
-    }
-
-
-    // TODO we should only use one rectangle class.
-    public boolean contains(com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle other) {
-
-        return page == other.getPage() && this.topLeft.getX() <= other.getTopLeftX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeftX() + other.getWidth() && this.getTopLeft()
-                .getY() <= other.getTopLeftY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeftY() + other.getHeight();
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java

@@ -1,33 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.dslplatform.json.CompiledJson;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.*;
-
-@Data
-@CompiledJson
-@AllArgsConstructor
-@NoArgsConstructor
-public class SectionGrid {
-
-    private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();
-
-    private List<SectionGridSection> sections = new ArrayList<>();
-
-    @Data
-    @NoArgsConstructor
-    @AllArgsConstructor
-    public static class SectionGridSection {
-
-        private int sectionNumber;
-        private String headline;
-        private Set<Integer> pages;
-        private List<SectionArea> sectionAreas;
-
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.util.List;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-public class SectionRectangle {
-
-    private Point topLeft;
-    private float width;
-    private float height;
-    private int part;
-    private int numberOfParts;
-
-    private List<CellRectangle> tableCells;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -1,38 +1,12 @@
 package com.iqser.red.service.redaction.v1.resources;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-
 import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
 public interface RedactionResource {
 
-    @PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/debug/sections", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult sections(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
-
-
     @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
     void testRules(@RequestBody String rules);
 
-
-    @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE)
-    RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest);
-
-
-    @PostMapping(value = "/manual/surrounding-text/{dossierId}/{fileId}", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
-    ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions);
-
 }

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <artifactId>redaction-service-v1</artifactId>
         <groupId>com.iqser.red.service</groupId>
-        <version>1.0-SNAPSHOT</version>
+        <version>4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>redaction-service-server-v1</artifactId>
@@ -30,7 +30,6 @@
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>storage-commons</artifactId>
-            <version>1.8.5</version>
         </dependency>
 
         <dependency>
@@ -94,6 +93,7 @@
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>metric-commons</artifactId>
+            <version>1.7.0</version>
         </dependency>
         <!-- other external -->
         <dependency>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -1,9 +1,5 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -13,10 +9,16 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig;
+import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
+import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyWebConfiguration;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
-@Import({DefaultWebMvcConfiguration.class})
+@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class, MetricsConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/MetricsConfiguration.java

@@ -0,0 +1,26 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+import com.iqser.gin4.commons.metrics.MetricCommonsConfiguration;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerFactory;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
+
+@Configuration
+@Import({MetricCommonsConfiguration.class})
+public class MetricsConfiguration {
+
+    private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
+
+
+    @Bean
+    @Qualifier(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME)
+    public FunctionTimerValues redactmanagerAnalyzePagewise(FunctionTimerFactory factory) {
+
+        return factory.create(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
 
 import lombok.Data;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java

@@ -3,9 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.pdfbox.pdmodel.common.PDRectangle;
-
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java

@@ -4,8 +4,8 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -5,7 +5,7 @@ import java.util.regex.Pattern;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.classification.utils;
 
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 
 import lombok.experimental.UtilityClass;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/RulingTextDirAdjustUtil.java

@@ -25,16 +25,13 @@ public final class RulingTextDirAdjustUtil {
     private Point2D convertPoint(float x, float y, float dir, float pageWidth, float pageHeight) {
 
         var xAdj = getXRot(x, y, dir, pageWidth, pageHeight);
-        var yAdj = 0f;
-        if (dir == 0 || dir == 180) {
-            yAdj = pageHeight - getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
-        } else {
-            yAdj = pageWidth - getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
-        }
+        var yLowerLeftRot = getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
+        var yAdj = dir == 0 || dir == 180 ? pageHeight - yLowerLeftRot : pageWidth - yLowerLeftRot;
         return new Point2D.Float(xAdj, yAdj);
     }
 
 
+    @SuppressWarnings("SuspiciousNameCombination")
     private float getXRot(float x, float y, float dir, float pageWidth, float pageHeight) {
 
         if (dir == 0) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/DictionaryClient.java

@@ -2,9 +2,9 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.DictionaryResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.DictionaryResource;
 
 @FeignClient(name = "DictionaryResource", url = "${persistence-service.url}")
 public interface DictionaryClient extends DictionaryResource {
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/EntityRecognitionClient.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client;
-
-import org.springframework.cloud.openfeign.FeignClient;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PostMapping;
-
-import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
-
-@FeignClient(name = "EntityRecognitionClient", url = "${entity-recognition-service.url}")
-public interface EntityRecognitionClient {
-
-    @PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
-    NerEntities findAuthors(EntityRecognitionRequest entityRecognitionRequest);
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/FileStatusProcessingUpdateClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
 
 @FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
 public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/LegalBasisClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.LegalBasisMappingResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.LegalBasisMappingResource;
 
 @FeignClient(name = "LegalBasisMappingResource", url = "${persistence-service.url}")
 public interface LegalBasisClient extends LegalBasisMappingResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/RulesClient.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.client;
 
 import org.springframework.cloud.openfeign.FeignClient;
 
-import com.iqser.red.service.persistence.service.v1.api.resources.RulesResource;
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.RulesResource;
 
 @FeignClient(name = "RulesResource", url = "${persistence-service.url}")
 public interface RulesClient extends RulesResource {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/TenantsClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
+
+@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
+public interface TenantsClient extends TenantsResource {
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionRequest.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client.model;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class EntityRecognitionRequest {
-
-    private List<EntityRecognitionSection> data;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionResult.java

@@ -1,21 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.client.model;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class EntityRecognitionResult {
-
-    @Builder.Default
-    private Map<Integer, List<EntityRecogintionEntity>> entities = new HashMap<>();
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,31 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -35,100 +15,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
 
-    private final PdfVisualisationService pdfVisualisationService;
     private final DroolsExecutionService droolsExecutionService;
-    private final PdfSegmentationService pdfSegmentationService;
-    private final RedactionStorageService redactionStorageService;
-    private final RedactionLogMergeService redactionLogMergeService;
-    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
-
-
-    @Override
-    public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                redactionRequest.getFileId(),
-                FileType.ORIGIN));
-        try {
-            Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-
-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
-                pdDocument.setAllSecurityToBeRemoved(true);
-
-                pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
-
-                return convert(pdDocument, classifiedDoc.getPages().size());
-
-            } catch (IOException e) {
-                throw new RedactionException(e);
-            }
-
-        } catch (IOException e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-    @Override
-    public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
-
-        var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                redactionRequest.getFileId(),
-                FileType.ORIGIN));
-        try {
-            Document classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-
-            storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
-                pdDocument.setAllSecurityToBeRemoved(true);
-
-                pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
-                return convert(pdDocument, classifiedDoc.getPages().size());
-
-            } catch (IOException e) {
-                throw new RedactionException(e);
-            }
-
-        } catch (IOException e) {
-            throw new RedactionException(e);
-        }
-
-    }
-
-
-    @Override
-    public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
-
-        Document classifiedDoc;
-
-        try {
-            var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(),
-                    redactionRequest.getFileId(),
-                    FileType.ORIGIN));
-            classifiedDoc = pdfSegmentationService.parseDocument(redactionRequest.getDossierId(), redactionRequest.getFileId(), storedObjectStream, null);
-        } catch (Exception e) {
-            throw new RedactionException(e);
-        }
-
-        StringBuilder sb = new StringBuilder();
-        for (Page page : classifiedDoc.getPages()) {
-            for (AbstractTextContainer textContainer : page.getTextBlocks()) {
-                if (textContainer instanceof Table) {
-                    Table table = (Table) textContainer;
-                    sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
-                }
-            }
-        }
-
-        return RedactionResult.builder().document(sb.toString().getBytes()).build();
-
-    }
 
 
     @Override
@@ -141,30 +28,4 @@ public class RedactionController implements RedactionResource {
         }
     }
 
-
-    @Override
-    public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
-
-        return redactionLogMergeService.provideRedactionLog(redactionRequest);
-    }
-
-
-    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
-
-        try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
-            document.save(byteArrayOutputStream);
-            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
-        }
-
-    }
-
-
-    @Override
-    public ManualRedactions addSurroundingText(@PathVariable("dossierId") String dossierId, @PathVariable("fileId") String fileId, @RequestBody ManualRedactions manualRedactions) {
-
-        var result = manualRedactionSurroundingTextService.addSurroundingText(dossierId, fileId, manualRedactions);
-        log.info("Added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", dossierId, fileId, result.getDuration());
-        return result.getManualRedactions();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/AsyncConfig.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import java.util.concurrent.Executor;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+@Configuration
+public class AsyncConfig extends AsyncConfigurerSupport {
+
+    @Override
+    public Executor getAsyncExecutor() {
+
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+
+        executor.setCorePoolSize(7);
+        executor.setMaxPoolSize(42);
+        executor.setQueueCapacity(11);
+        executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
+        executor.setTaskDecorator(new TenantAwareTaskDecorator());
+        executor.initialize();
+
+        return executor;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java

@@ -0,0 +1,105 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
+import java.security.spec.KeySpec;
+import java.util.Base64;
+
+import javax.annotation.PostConstruct;
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import lombok.SneakyThrows;
+
+@Service
+public class EncryptionDecryptionService {
+
+    @Value("${redaction-service.crypto.key:redaction}")
+    private String key;
+
+    private SecretKey secretKey;
+    private byte[] iv;
+
+
+    @SneakyThrows
+    @PostConstruct
+    protected void postConstruct() {
+
+        SecureRandom secureRandom = new SecureRandom();
+        iv = new byte[12];
+        secureRandom.nextBytes(iv);
+        secretKey = generateSecretKey(key, iv);
+    }
+
+
+    @SneakyThrows
+    public String encrypt(String strToEncrypt) {
+
+        return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
+    }
+
+
+    @SneakyThrows
+    public String decrypt(String strToDecrypt) {
+
+        byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
+        return new String(decrypt(bytes), StandardCharsets.UTF_8);
+    }
+
+
+    @SneakyThrows
+    public byte[] encrypt(byte[] data) {
+
+        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
+        cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
+        byte[] encryptedData = cipher.doFinal(data);
+        ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
+        byteBuffer.putInt(iv.length);
+        byteBuffer.put(iv);
+        byteBuffer.put(encryptedData);
+        return byteBuffer.array();
+    }
+
+
+    @SneakyThrows
+    public byte[] decrypt(byte[] encryptedData) {
+
+        ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
+        int noonceSize = byteBuffer.getInt();
+        if (noonceSize < 12 || noonceSize >= 16) {
+            throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
+        }
+        byte[] iv = new byte[noonceSize];
+        byteBuffer.get(iv);
+
+        SecretKey secretKey = generateSecretKey(key, iv);
+
+        byte[] cipherBytes = new byte[byteBuffer.remaining()];
+        byteBuffer.get(cipherBytes);
+
+        Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
+        GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
+        cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
+        return cipher.doFinal(cipherBytes);
+    }
+
+
+    @SneakyThrows
+    public SecretKey generateSecretKey(String password, byte[] iv) {
+
+        KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
+        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
+        byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
+        return new SecretKeySpec(key, "AES");
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/ForwardTenantInterceptor.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Component;
+
+import feign.RequestInterceptor;
+import feign.RequestTemplate;
+
+@Component
+public class ForwardTenantInterceptor implements RequestInterceptor {
+
+    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
+
+    @Override
+    public void apply(RequestTemplate template) {
+        template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java

@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+
+import static com.iqser.red.service.redaction.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
+
+import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.config.BeanPostProcessor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class MultiTenancyMessagingConfiguration {
+
+    @Bean
+    public static BeanPostProcessor multitenancyBeanPostProcessor() {
+
+        return new BeanPostProcessor() {
+
+            @Override
+            public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
+
+                if (bean instanceof RabbitTemplate) {
+
+                    ((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
+                        m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
+                        return m;
+                    });
+
+                } else if (bean instanceof AbstractRabbitListenerContainerFactory) {
+
+                    ((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
+                        String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
+
+                        if (tenant != null) {
+                            TenantContext.setTenantId(tenant);
+                        } else {
+                            throw new RuntimeException("No Tenant is set queue message");
+                        }
+                        return m;
+                    });
+                }
+                return bean;
+            }
+        };
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyWebConfiguration.java

@@ -0,0 +1,28 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
+
+import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
+
+@Configuration
+public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
+
+    private final TenantInterceptor tenantInterceptor;
+
+
+    @Autowired
+    public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
+
+        this.tenantInterceptor = tenantInterceptor;
+    }
+
+
+    @Override
+    public void addInterceptors(InterceptorRegistry registry) {
+
+        registry.addWebRequestInterceptor(tenantInterceptor);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/StorageConnectionProviderImpl.java

@@ -0,0 +1,45 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.client.TenantsClient;
+import com.iqser.red.storage.commons.model.AzureStorageConnection;
+import com.iqser.red.storage.commons.model.S3StorageConnection;
+import com.iqser.red.storage.commons.service.StorageConnectionProvider;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class StorageConnectionProviderImpl implements StorageConnectionProvider {
+
+    private final TenantsClient tenantsClient;
+    private final EncryptionDecryptionService encryptionDecryptionService;
+
+
+    @Override
+    public AzureStorageConnection getAzureStorageConnection(String tenantId) {
+
+        var tenant = tenantsClient.getTenant(tenantId);
+        return AzureStorageConnection.builder()
+                .connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
+                .containerName(tenant.getAzureStorageConnection().getContainerName())
+                .build();
+    }
+
+
+    @Override
+    public S3StorageConnection getS3StorageConnection(String tenantId) {
+
+        var tenant = tenantsClient.getTenant(tenantId);
+        return S3StorageConnection.builder()
+                .key(tenant.getS3StorageConnection().getKey())
+                .secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
+                .signerType(tenant.getS3StorageConnection().getSignerType())
+                .bucketName(tenant.getS3StorageConnection().getBucketName())
+                .region(tenant.getS3StorageConnection().getRegion())
+                .endpoint(tenant.getS3StorageConnection().getEndpoint())
+                .build();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantAwareTaskDecorator.java

@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.core.task.TaskDecorator;
+import org.springframework.lang.NonNull;
+
+public class TenantAwareTaskDecorator implements TaskDecorator {
+
+    @Override
+    @NonNull
+    public Runnable decorate(@NonNull Runnable runnable) {
+
+        String tenantId = TenantContext.getTenantId();
+        return () -> {
+            try {
+                TenantContext.setTenantId(tenantId);
+                runnable.run();
+            } finally {
+                TenantContext.setTenantId(null);
+            }
+        };
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantContext.java

@@ -0,0 +1,29 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public final class TenantContext {
+
+    private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
+
+
+    public static void setTenantId(String tenantId) {
+
+        log.debug("Setting tenantId to " + tenantId);
+        currentTenant.set(tenantId);
+    }
+
+
+    public static String getTenantId() {
+
+        return currentTenant.get();
+    }
+
+
+    public static void clear() {
+
+        currentTenant.remove();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantInterceptor.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.redaction.v1.server.multitenancy;
+
+import org.springframework.stereotype.Component;
+import org.springframework.ui.ModelMap;
+import org.springframework.web.context.request.WebRequest;
+import org.springframework.web.context.request.WebRequestInterceptor;
+
+@Component
+public class TenantInterceptor implements WebRequestInterceptor {
+
+    public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
+
+
+    @Override
+    public void preHandle(WebRequest request) {
+
+        if (request.getHeader(TENANT_HEADER_NAME) != null) {
+            TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
+        }
+    }
+
+
+    @Override
+    public void postHandle(WebRequest request, ModelMap model) {
+
+        TenantContext.clear();
+    }
+
+
+    @Override
+    public void afterCompletion(WebRequest request, Exception ex) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -12,8 +12,8 @@ import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -225,9 +225,6 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public String getFont() {
 
-        if (textPositions.get(0).getFontName() == null) {
-            return "none";
-        }
         return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
     }
 
@@ -236,9 +233,6 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public String getFontStyle() {
 
-        if (textPositions.get(0).getFontName() == null) {
-            return "standard";
-        }
         String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
 
         if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -10,12 +10,12 @@ import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java

@@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 import java.util.HashSet;
 import java.util.Set;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
 import lombok.*;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/FindEntitiesResult.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
 
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PageEntities.java

@@ -6,7 +6,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -18,11 +18,11 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -1589,7 +1589,7 @@ public class Section {
         if (values != null) {
             for (String value : values) {
                 if (StringUtils.isNotBlank(value)) {
-                    String[] lines = value.trim().split("\n");
+                    String[] lines = value.split("\n");
 
                     for (String line : lines) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class TenantDictionary {
+
+    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
+    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,25 +1,46 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import java.awt.Color;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.SerializationUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
 
 import feign.FeignException;
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.SerializationUtils;
-import org.springframework.stereotype.Service;
-
-import java.awt.*;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -28,22 +49,40 @@ public class DictionaryService {
     private static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
-    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
-    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+    @Value("${multitenancy.dictionary-cache.maximumSize:100}")
+    private Long maximumSize;
+
+    @Value("${multitenancy.dictionary-cache.expireAfterAccess:3}")
+    private Integer expireAfterAccess;
+
+    private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
 
 
+    @PostConstruct
+    protected void createCache() {
+
+        tenantDictionaryCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(expireAfterAccess, TimeUnit.DAYS).build(new CacheLoader<>() {
+            public TenantDictionary load(String key) {
+
+                return new TenantDictionary();
+            }
+        });
+    }
+
+
+    @SneakyThrows
     @Timed("redactmanager_updateDictionary")
     public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
 
         log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
         long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
-        var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
         if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
         }
 
         long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
-        var dossierDictionary = dictionariesByDossier.get(dossierId);
+        var dossierDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
         if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
         }
@@ -52,13 +91,14 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDictionaryIncrements")
     public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
 
         DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
 
         Set<DictionaryIncrementValue> newValues = new HashSet<>();
-        List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
+        List<DictionaryModel> dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDictionary();
         dictionaryModels.forEach(dictionaryModel -> {
             dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                 if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@@ -77,8 +117,8 @@ public class DictionaryService {
             });
         });
 
-        if (dictionariesByDossier.containsKey(dossierId)) {
-            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
+        if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) {
+            dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId).getDictionary();
             dictionaryModels.forEach(dictionaryModel -> {
                 dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                     if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
@@ -102,6 +142,7 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {
 
         try {
@@ -115,10 +156,20 @@ public class DictionaryService {
 
                     Optional<DictionaryModel> oldModel;
                     if (dossierId == null) {
-                        var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
+                        DictionaryRepresentation representation = null;
+                        try {
+                            representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
+                        } catch (ExecutionException e) {
+                            throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId());
+                        }
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     } else {
-                        var representation = dictionariesByDossier.get(dossierId);
+                        DictionaryRepresentation representation = null;
+                        try {
+                            representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
+                        } catch (ExecutionException e) {
+                            throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId());
+                        }
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     }
 
@@ -178,9 +229,9 @@ public class DictionaryService {
                 dictionaryRepresentation.setDictionary(dictionary);
 
                 if (dossierId == null) {
-                    dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
+                    tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation);
                 } else {
-                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
+                    tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation);
                 }
             }
         } catch (FeignException e) {
@@ -219,19 +270,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getColor(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.getColor();
         }
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDefaultColor();
     }
 
 
+    @SneakyThrows
     public boolean isHint(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.isHint();
         }
@@ -239,20 +292,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDeepCopyDictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
         List<DictionaryModel> copy = new ArrayList<>();
 
-        var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
         dossierTemplateRepresentation.getDictionary().forEach(dm -> {
             copy.add(SerializationUtils.clone(dm));
         });
 
         //TODO merge dictionaries if they have same names
         long dossierDictionaryVersion = -1;
-        if (dictionariesByDossier.containsKey(dossierId)) {
-            var dossierRepresentation = dictionariesByDossier.get(dossierId);
+        if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) {
+            var dossierRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
             dossierRepresentation.getDictionary().forEach(dm -> {
                 copy.add(SerializationUtils.clone(dm));
             });
@@ -264,9 +318,10 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getNotRedactedColor(String dossierTemplateId) {
 
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getNotRedactedColor();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java

@@ -5,11 +5,11 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.ImportedRedaction;
-import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -51,8 +51,8 @@ public class ImportedRedactionService {
                                                                              List<RedactionLogEntry> redactionLogEntries,
                                                                              ImportedRedactions importedRedactions) {
 
-        for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
-            for (ImportedRedaction importedRedaction : importedRedactionsValues) {
+        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
+            for (var importedRedaction : importedRedactionsValues) {
                 RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
                         .id(importedRedaction.getId())
                         .type(IMPORTED_REDACTION_TYPE)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -7,12 +7,11 @@ import java.util.Set;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -23,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
@@ -125,7 +125,7 @@ public class ManualRedactionSurroundingTextService {
     }
 
 
-    public boolean intersects(Rectangle manualPosition, com.iqser.red.service.redaction.v1.model.Rectangle textPositionRectangle) {
+    public boolean intersects(Rectangle manualPosition, com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle textPositionRectangle) {
 
         return textPositionRectangle.getTopLeft().getX() + textPositionRectangle.getWidth() > manualPosition.getTopLeftX() && textPositionRectangle.getTopLeft()
                 .getY() + textPositionRectangle.getHeight() > manualPosition.getTopLeftY() && textPositionRectangle.getTopLeft()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -9,9 +9,9 @@ import java.util.Set;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -1,585 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import java.awt.Color;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.Change;
-import com.iqser.red.service.redaction.v1.model.ChangeType;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.ManualChange;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogComment;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class RedactionLogMergeService {
-
-    private static final String DELETED_TYPE_COLOR = "#9398a0";
-    private final SectionTextService sectionTextService;
-
-    private final RedactionStorageService redactionStorageService;
-
-
-    @Timed("redactmanager_getMergedRedactionLog")
-    public RedactionLog provideRedactionLog(RedactionRequest redactionRequest) {
-
-        log.debug("Requested preview for: {}", redactionRequest);
-
-        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-        if (redactionLog == null) {
-            throw new NotFoundException("RedactionLog not present");
-        }
-
-        SectionGrid sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId());
-        if (sectionGrid.getSections().isEmpty()) {
-
-            log.info("SectionGrid does not have headlines set. Computing headlines now!");
-            var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId());
-
-            // enhance section grid with headline data
-            for (var sectionText : text.getSectionTexts()) {
-                sectionGrid.getSections()
-                        .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(),
-                                sectionText.getHeadline(),
-                                sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
-                                sectionText.getSectionAreas()));
-            }
-            redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
-        }
-
-        log.debug("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        var merged = mergeRedactionLogData(redactionLog,
-                sectionGrid,
-                redactionRequest.getManualRedactions(),
-                redactionRequest.getExcludedPages(),
-                redactionRequest.getTypes(),
-                redactionRequest.getColors());
-
-        merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
-
-        return merged;
-    }
-
-
-    private RedactionLog mergeRedactionLogData(RedactionLog redactionLog,
-                                               SectionGrid sectionGrid,
-                                               ManualRedactions manualRedactions,
-                                               Set<Integer> excludedPages,
-                                               List<Type> types,
-                                               Colors colors) {
-
-        var skippedImportedRedactions = new HashSet<>();
-        log.info("Merging Redaction log with manual redactions");
-        if (manualRedactions != null) {  // This is never null! Could be removed
-
-            var manualRedactionLogEntries = addManualAddEntries(sectionGrid,
-                    manualRedactions.getEntriesToAdd(),
-                    manualRedactions.getComments(),
-                    colors,
-                    types,
-                    redactionLog.getAnalysisNumber());
-
-            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
-
-            var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
-
-            for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-
-                processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList()), entry, types, colors);
-
-                if (entry.isImported() && !entry.isRedacted()) {
-                    skippedImportedRedactions.add(entry.getId());
-                }
-
-                entry.setComments(convert(manualRedactions.getComments().get(entry.getId())));
-
-                if (excludedPages != null && !excludedPages.isEmpty()) {
-                    entry.getPositions().forEach(pos -> {
-                        if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
-                            entry.setExcluded(true);
-                        } else {
-                            entry.setExcluded(false); // This is a workaround, because in the analysis exlcuded is also set, but should not, because it triggers no reanalyis
-                        }
-                    });
-                } else {
-                    entry.setExcluded(false); // This is a workaround, because in the analysis exlcuded is also set, but should not, because it triggers no reanalyis
-                }
-            }
-
-        }
-
-        Set<String> processedIds = new HashSet<>();
-        redactionLog.getRedactionLogEntry().removeIf(entry -> {
-
-            if (entry.getImportedRedactionIntersections() != null) {
-                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
-                        .equals("ocr")))) {
-                    return true;
-                }
-            }
-            if (processedIds.contains(entry.getId())) {
-                log.info("Duplicate annotation found with id {}", entry.getId());
-                return true;
-            }
-            processedIds.add(entry.getId());
-            return false;
-        });
-        return redactionLog;
-    }
-
-
-    private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
-
-        List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
-
-        manualRedactions.getImageRecategorization().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getIdsToRemove().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getForceRedactions().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getLegalBasisChanges().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        manualRedactions.getResizeRedactions().forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
-
-        Collections.sort(manualRedactionWrappers);
-
-        return manualRedactionWrappers;
-    }
-
-
-    private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {
-
-        manualRedactionWrappers.forEach(mrw -> {
-
-            Object item = mrw.getItem();
-            if (item instanceof ManualImageRecategorization) {
-                var imageRecategorization = (ManualImageRecategorization) item;
-                processManualImageRecategorization(redactionLogEntry, types, colors, imageRecategorization);
-            }
-
-            if (item instanceof IdRemoval) {
-                var manualRemoval = (IdRemoval) item;
-                processIdRemoval(redactionLogEntry, types, colors, manualRemoval);
-            }
-
-            if (item instanceof ManualForceRedaction) {
-                var manualForceRedact = (ManualForceRedaction) item;
-                processManualForceRedaction(redactionLogEntry, types, colors, manualForceRedact);
-            }
-
-            if (item instanceof ManualLegalBasisChange) {
-                var manualLegalBasisChange = (ManualLegalBasisChange) item;
-                processManualLegalBasisChange(redactionLogEntry, types, colors, manualLegalBasisChange);
-            }
-
-            if (item instanceof ManualResizeRedaction) {
-                var manualResizeRedact = (ManualResizeRedaction) item;
-                processManualResizeRedaction(redactionLogEntry, types, colors, manualResizeRedact);
-            }
-
-        });
-
-    }
-
-
-    private void processManualImageRecategorization(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualImageRecategorization imageRecategorization) {
-
-        String manualOverrideReason = null;
-        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-
-            redactionLogEntry.setType(imageRecategorization.getType());
-            redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
-
-            if (isHint(types, imageRecategorization.getType())) {
-                redactionLogEntry.setRedacted(false);
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setHint(false);
-            }
-
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
-        } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
-    }
-
-
-    private String mergeReasonIfNecessary(String currentReason, String addition) {
-
-        if (currentReason != null) {
-            if (!currentReason.contains(addition)) {
-                return currentReason + addition;
-            }
-            return currentReason;
-        } else {
-            return "";
-        }
-    }
-
-
-    private void processIdRemoval(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, IdRemoval manualRemoval) {
-
-        boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
-        if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && manualRemoval.getProcessedDate() != null && isBasedOnDictionaryOnly(redactionLogEntry)) {
-            log.debug("Skipping merge for dictionary-modifying entry");
-        } else {
-            String redactionLogEntryType = redactionLogEntry.getType();
-            String manualOverrideReason = null;
-            if (isApprovedRedaction) {
-                redactionLogEntry.setRedacted(false);
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                redactionLogEntry.setColor(getColor(redactionLogEntryType, colors, false, redactionLogEntry.isRedacted(), true, types));
-                redactionLogEntry.setHint(false);
-            } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                redactionLogEntry.setColor(getColor(redactionLogEntryType, colors, true, redactionLogEntry.isRedacted(), false, types));
-            }
-
-            if (manualOverrideReason != null) {
-                redactionLogEntry.setReason(manualOverrideReason);
-            }
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualRemoval)
-                        .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
-    }
-
-
-    private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
-
-        return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
-    }
-
-
-    private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualForceRedaction manualForceRedact) {
-
-        String manualOverrideReason = null;
-        var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
-        if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            // Forcing a skipped hint should result in a hint
-            if (dictionaryIsHint) {
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setRedacted(true);
-            }
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
-
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualLegalBasisChange manualLegalBasisChange) {
-
-        String manualOverrideReason = null;
-        if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
-            redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-            redactionLogEntry.setRedacted(true);
-            if (manualLegalBasisChange.getSection() != null) {
-                redactionLogEntry.setSection(manualLegalBasisChange.getSection());
-            }
-            if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-            }
-        } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
-        manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
-        if (manualLegalBasisChange.getSection() != null) {
-            manualChange.withChange("section", manualLegalBasisChange.getSection());
-        }
-        if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-            manualChange.withChange("value", manualLegalBasisChange.getValue());
-        }
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors, ManualResizeRedaction manualResizeRedact) {
-
-        String manualOverrideReason = null;
-        if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-            if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
-                redactionLogEntry.setValue(manualResizeRedact.getValue());
-            }
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
-        } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-            redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-        }
-
-        redactionLogEntry.setReason(manualOverrideReason);
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
-    }
-
-
-    public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid,
-                                                       Set<ManualRedactionEntry> manualAdds,
-                                                       Map<String, List<Comment>> comments,
-                                                       Colors colors,
-                                                       List<Type> types,
-                                                       int analysisNumber) {
-
-        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
-
-        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
-
-            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types, analysisNumber);
-                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId())));
-                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
-                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
-
-                sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<RedactionLogComment> convert(List<Comment> comments) {
-
-        return comments == null ? null : comments.stream()
-                .map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), c.getFileId(), c.getDate(), c.getSoftDeletedTime()))
-                .collect(Collectors.toList());
-    }
-
-
-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle> positions) {
-
-        return positions.stream()
-                .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
-                .collect(Collectors.toList());
-    }
-
-
-    @SuppressWarnings("PMD.UselessParentheses")
-    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, Colors colors, List<Type> types, int analysisNumber) {
-
-        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
-
-        var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
-        List<ManualChange> changeList = new ArrayList<>();
-        changeList.add(change);
-
-        return RedactionLogEntry.builder()
-                .id(id)
-                .color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
-                .reason(manualRedactionEntry.getReason())
-                .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
-                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .value(manualRedactionEntry.getValue())
-                .sourceId(manualRedactionEntry.getSourceId())
-                .section(manualRedactionEntry.getSection())
-                .type(manualRedactionEntry.getType())
-                .redacted(true)
-                .isHint(false)
-                .sectionNumber(-1)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualChanges(changeList)
-                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
-                .build();
-    }
-
-
-    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped, List<Type> types) {
-
-        if (requested) {
-            return convertColor(colors.getRequestRemoveColor());
-        }
-        if (skipped || !isRedaction && !isHint(types, type)) {
-            return convertColor(colors.getSkippedColor());
-        }
-        return getColor(types, type);
-    }
-
-
-    private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
-
-        if (status.equals(AnnotationStatus.REQUESTED)) {
-            return convertColor(colors.getRequestAddColor());
-        } else if (status.equals(AnnotationStatus.DECLINED)) {
-            return convertColor(colors.getSkippedColor());
-        }
-        return getColor(types, type);
-    }
-
-
-    private float[] getColor(List<Type> types, String type) {
-
-        var matchingTypes = getMatchingTypes(types, type);
-        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
-        if (foundAndNotDeletedType.isPresent()) {
-            return convertColor(foundAndNotDeletedType.get().getHexColor());
-        }
-        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
-    }
-
-
-    boolean isHint(List<Type> types, String type) {
-
-        var matchingTypes = getMatchingTypes(types, type);
-        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
-        if (foundAndNotDeletedType.isPresent()) {
-            return foundAndNotDeletedType.get().isHint();
-        }
-        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
-        return firstDeletedType.map(Type::isHint).orElse(false);
-    }
-
-
-    private float[] convertColor(String hex) {
-
-        Color color = Color.decode(hex);
-        return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
-    }
-
-
-    private List<Type> getMatchingTypes(List<Type> types, String type) {
-
-        return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
-    }
-
-
-    private boolean isDeletedType(Type type) {
-
-        return type.getSoftDeletedTime() != null;
-    }
-
-
-    @Data
-    @AllArgsConstructor
-    private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
-
-        private String id;
-        private OffsetDateTime date;
-        private Object item;
-
-
-        @Override
-        public int compareTo(ManualRedactionWrapper o) {
-
-            return this.date.compareTo(o.date);
-        }
-
-    }
-
-}
-

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.model.CellRectangle;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -11,8 +11,8 @@ import java.util.stream.Collectors;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextService.java

@@ -1,41 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
-
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Service;
-
-@Slf4j
-@Service
-public class SectionTextService {
-
-    public void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.getSection() != null) {
-            // set by UI
-            return;
-        }
-
-        if (sectionGrid != null) {
-            var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null;
-
-            if (firstPosition != null) {
-
-                for (var section : sectionGrid.getSections()) {
-                    if (section.getPages().contains(firstPosition.getPage())) {
-                        for (var sectionArea : section.getSectionAreas()) {
-                            if (sectionArea.contains(firstPosition)) {
-                                redactionLogEntry.setSection(section.getHeadline());
-                                redactionLogEntry.setSectionNumber(section.getSectionNumber());
-                                return;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -1,38 +1,30 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
 
 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
@@ -44,42 +36,53 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionChangeLogService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SectionGridCreatorService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SectionTextBuilderService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction.EntityRedactionService;
 import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 @RequiredArgsConstructor
 public class AnalyzeService {
 
-    private final DictionaryService dictionaryService;
-    private final DroolsExecutionService droolsExecutionService;
-    private final EntityRedactionService entityRedactionService;
-    private final RedactionLogCreatorService redactionLogCreatorService;
-    private final RedactionStorageService redactionStorageService;
-    private final PdfSegmentationService pdfSegmentationService;
-    private final RedactionChangeLogService redactionChangeLogService;
-    private final LegalBasisClient legalBasisClient;
-    private final RedactionServiceSettings redactionServiceSettings;
-    private final SectionTextBuilderService sectionTextBuilderService;
-    private final SectionGridCreatorService sectionGridCreatorService;
-    private final ImageService imageService;
-    private final ImportedRedactionService importedRedactionService;
+    private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
+
+    DictionaryService dictionaryService;
+    DroolsExecutionService droolsExecutionService;
+    EntityRedactionService entityRedactionService;
+    RedactionLogCreatorService redactionLogCreatorService;
+    RedactionStorageService redactionStorageService;
+    PdfSegmentationService pdfSegmentationService;
+    RedactionChangeLogService redactionChangeLogService;
+    LegalBasisClient legalBasisClient;
+    RedactionServiceSettings redactionServiceSettings;
+    SectionTextBuilderService sectionTextBuilderService;
+    SectionGridCreatorService sectionGridCreatorService;
+    ImageService imageService;
+    ImportedRedactionService importedRedactionService;
+    SectionFinder sectionFinder;
+
+    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
 
     @Timed("redactmanager_analyzeDocumentStructure")
@@ -153,32 +156,25 @@ public class AnalyzeService {
                 new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
                 analyzeRequest.getDossierId());
 
-        Set<Integer> sectionsToReanalyse = !analyzeRequest.getSectionsToReanalyse().isEmpty() ? analyzeRequest.getSectionsToReanalyse() : findSectionsToReanalyse(
-                dictionaryIncrement,
-                redactionLog,
-                text,
-                analyzeRequest);
+        Set<Integer> sectionsToReanalyse = analyzeRequest.getSectionsToReanalyse().isEmpty() //
+                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest) //
+                : analyzeRequest.getSectionsToReanalyse();
+        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
 
         if (sectionsToReanalyse.isEmpty()) {
             return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
         }
 
-        NerEntities nerEntities;
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        } else {
-            nerEntities = new NerEntities();
-        }
-
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         List<SectionText> reanalysisSections = text.getSectionTexts()
                 .stream()
                 .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
                 .collect(Collectors.toList());
-
+        NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() //
+                ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) //
+                : new NerEntities();
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
 
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
         PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
 
         var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
@@ -238,48 +234,13 @@ public class AnalyzeService {
     }
 
 
-    @Timed("redactmanager_findSectionsToReanalyse")
-    private Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
-
-        long start = System.currentTimeMillis();
-        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
-
-        Set<Integer> sectionsToReanalyse = new HashSet<>();
-        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
-        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                sectionsToReanalyse.add(entry.getSectionNumber());
-            }
-            if (entry.isImage()) {
-                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
-            }
-        }
-
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
-
-        for (SectionText sectionText : text.getSectionTexts()) {
-
-            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
-                sectionsToReanalyse.add(sectionText.getSectionNumber());
-            }
-
-        }
-
-        log.info("Should reanalyze {} sections for request: {}, took: {}", sectionsToReanalyse.size(), analyzeRequest, System.currentTimeMillis() - start);
-
-        return sectionsToReanalyse;
-    }
-
-
     private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                            long startTime,
                                            RedactionLog redactionLog,
                                            Text text,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
-                                           Set<FileAttribute> addedFileAttributes
-    ) {
+                                           Set<FileAttribute> addedFileAttributes) {
 
         redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
         redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
@@ -294,6 +255,8 @@ public class AnalyzeService {
 
         long duration = System.currentTimeMillis() - startTime;
 
+        redactmanagerAnalyzePagewiseValues.increase(text.getNumberOfPages(), duration);
+
         return AnalyzeResult.builder()
                 .dossierId(analyzeRequest.getDossierId())
                 .fileId(analyzeRequest.getFileId())
@@ -313,41 +276,12 @@ public class AnalyzeService {
     }
 
 
-    private Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
-
-        if (manualRedactions == null) {
-            return new HashSet<>();
-        }
-
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
-    }
-
-
     public List<RedactionLogLegalBasis> convert(List<LegalBasis> legalBasis) {
 
         return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
     }
 
 
-    public Image convert(RedactionLogEntry entry) {
-
-        Rectangle position = entry.getPositions().get(0);
-
-        return Image.builder()
-                .type(entry.getType())
-                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
-                .sectionNumber(entry.getSectionNumber())
-                .section(entry.getSection())
-                .page(position.getPage())
-                .hasTransparency(entry.isImageHasTransparency())
-                .build();
-    }
-
-
     private void excludeExcludedPages(RedactionLog redactionLog, Set<Integer> excludedPages) {
 
         if (excludedPages != null && !excludedPages.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java

@@ -0,0 +1,101 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service.analyze;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.Text;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+class SectionFinder {
+    @Timed("redactmanager_findSectionsToReanalyse")
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
+
+        long start = System.currentTimeMillis();
+        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+
+        Set<Integer> sectionsToReanalyse = new HashSet<>();
+        Map<Integer, Set<Image>> imageEntries = new HashMap<>();
+        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
+            if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
+                sectionsToReanalyse.add(entry.getSectionNumber());
+            }
+            if (entry.isImage()) {
+                imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
+            }
+        }
+
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
+                true);
+
+        for (SectionText sectionText : text.getSectionTexts()) {
+
+            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
+                sectionsToReanalyse.add(sectionText.getSectionNumber());
+            }
+
+        }
+
+        log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
+
+        return sectionsToReanalyse;
+    }
+
+    private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null) {
+            return new HashSet<>();
+        }
+
+        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
+                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
+                        Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId),
+                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
+                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+    }
+
+    private static Image convert(RedactionLogEntry entry) {
+
+        Rectangle position = entry.getPositions().get(0);
+
+        return Image.builder()
+                .type(entry.getType())
+                .position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft().getY(), position.getWidth(), position.getHeight()))
+                .sectionNumber(entry.getSectionNumber())
+                .section(entry.getSection())
+                .page(position.getPage())
+                .hasTransparency(entry.isImageHasTransparency())
+                .build();
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java

@@ -0,0 +1,127 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+class EntityFinder {
+
+    RedactionServiceSettings redactionServiceSettings;
+
+
+    @Timed("redactmanager_findEntities")
+    public Entities findEntities(SearchableText searchableText,
+                                 String headline,
+                                 int sectionNumber,
+                                 Dictionary dictionary,
+                                 boolean local,
+                                 NerEntities nerEntities,
+                                 List<Integer> cellStarts,
+                                 ManualRedactions manualRedactions) {
+
+        Set<Entity> found = new HashSet<>();
+        String searchableString = searchableText.asString();
+
+        if (StringUtils.isEmpty(searchableString)) {
+            return new Entities(new HashSet<>(), new HashSet<>());
+        }
+
+        String lowercaseInputString = searchableString.toLowerCase();
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+
+            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation,
+                    model,
+                    new FindEntityDetails(model.getType(),
+                            headline,
+                            sectionNumber,
+                            !local,
+                            model.isDossierDictionary(),
+                            local ? Engine.RULE : Engine.DICTIONARY,
+                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
+        }
+
+        Set<Entity> nerFound = new HashSet<>();
+        if (!local) {
+            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
+        }
+
+        var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        return new Entities(cleared.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()), nerFound);
+    }
+
+
+    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
+
+        Set<Entity> entities = new HashSet<>();
+
+        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
+            nerEntities.getData().get(sectionNumber).forEach(res -> {
+                if (cellStarts == null || cellStarts.isEmpty()) {
+                    entities.add(new Entity(res.getValue(),
+                            res.getType(),
+                            res.getStartOffset(),
+                            res.getEndOffset(),
+                            headline,
+                            sectionNumber,
+                            false,
+                            false,
+                            Engine.NER,
+                            EntityType.RECOMMENDATION));
+                } else {
+                    boolean intersectsCellStart = false;
+                    for (Integer cellStart : cellStarts) {
+                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
+                            intersectsCellStart = true;
+                            break;
+                        }
+                    }
+                    if (!intersectsCellStart) {
+                        entities.add(new Entity(res.getValue(),
+                                res.getType(),
+                                res.getStartOffset(),
+                                res.getEndOffset(),
+                                headline,
+                                sectionNumber,
+                                false,
+                                false,
+                                Engine.NER,
+                                EntityType.RECOMMENDATION));
+                    }
+                }
+            });
+        }
+        return entities;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -1,41 +1,52 @@
-package com.iqser.red.service.redaction.v1.server.redaction.service;
+package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.commons.lang3.StringUtils;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityRedactionService {
 
-    private final RedactionServiceSettings redactionServiceSettings;
-    private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;
+    DroolsExecutionService droolsExecutionService;
+    SurroundingWordsService surroundingWordsService;
+    EntityFinder entityFinder;
 
 
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@@ -45,7 +56,7 @@ public class EntityRedactionService {
 
         if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
 
-            if(!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
+            if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
                 //AnalyzeRequest provides immutable list.
                 List<FileAttribute> mergedFileAttributes = new ArrayList<>();
                 mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
@@ -54,7 +65,14 @@ public class EntityRedactionService {
             }
 
             Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
-            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities);
+            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
+                    dictionary,
+                    kieContainer,
+                    analyzeRequest,
+                    true,
+                    hintsPerSectionNumber,
+                    imagesPerPage,
+                    nerEntities);
             EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
             EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
         }
@@ -67,18 +85,78 @@ public class EntityRedactionService {
 
 
     public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
-                                    Dictionary dictionary,
-                                    KieContainer kieContainer,
-                                    AnalyzeRequest analyzeRequest,
-                                    boolean local,
-                                    Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                    Map<Integer, Set<Image>> imagesPerPage,
-                                    NerEntities nerEntities) {
+                                           Dictionary dictionary,
+                                           KieContainer kieContainer,
+                                           AnalyzeRequest analyzeRequest,
+                                           boolean local,
+                                           Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                           Map<Integer, Set<Image>> imagesPerPage,
+                                           NerEntities nerEntities) {
 
-        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
-        for (SectionText reanalysisSection : reanalysisSections) {
+        List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
+                dictionary,
+                analyzeRequest,
+                local,
+                hintsPerSectionNumber,
+                nerEntities);
 
-            Entities entities = findEntities(reanalysisSection.getSearchableText(),
+        Set<FileAttribute> addedFileAttributes = new HashSet<>();
+        Set<Entity> entities = new HashSet<>();
+        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
+
+            if (!addedFileAttributes.isEmpty()) {
+                //Section.Builder provides immutable list.
+                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
+                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
+                mergedFileAttributes.addAll(addedFileAttributes);
+                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
+            }
+
+            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
+
+            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
+
+            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+
+            var entriesWithoutSurroundingText = analysedSection.getEntities()
+                    .stream()
+                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
+                    .collect(Collectors.toSet());
+
+            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                        sectionSearchableTextPair.getSearchableText(),
+                        dictionary,
+                        sectionSearchableTextPair.getCellStarts());
+            } else {
+                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+            }
+
+            entities.addAll(analysedSection.getEntities());
+
+            if (!local) {
+                for (Image image : analysedSection.getImages()) {
+                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
+                }
+                addLocalValuesToDictionary(analysedSection, dictionary);
+            }
+
+        });
+
+        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
+    }
+
+
+    private List<SectionSearchableTextPair> extractSearchableTextPairs(List<SectionText> reanalysisSections,
+                                                                       Dictionary dictionary,
+                                                                       AnalyzeRequest analyzeRequest,
+                                                                       boolean local,
+                                                                       Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                                                       NerEntities nerEntities) {
+
+        return reanalysisSections.stream().map(reanalysisSection -> {
+
+            Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
                     reanalysisSection.getHeadline(),
                     reanalysisSection.getSectionNumber(),
                     dictionary,
@@ -136,73 +214,36 @@ public class EntityRedactionService {
 
             log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
 
-            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
-                    .isLocal(false)
-                    .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
-                            .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                    .nerEntities(entities.getNerEntities())
-                    .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
-                    .searchText(reanalysisSection.getSearchableText().toString())
-                    .headline(reanalysisSection.getHeadline())
-                    .sectionNumber(reanalysisSection.getSectionNumber())
-                    .tabularData(reanalysisSection.getTabularData())
-                    .searchableText(reanalysisSection.getSearchableText())
-                    .dictionary(dictionary)
-                    .images(reanalysisSection.getImages())
-                    .sectionAreas(reanalysisSection.getSectionAreas())
-                    .fileAttributes(analyzeRequest.getFileAttributes())
-                    .manualRedactions(analyzeRequest.getManualRedactions())
-                    .isInTable(reanalysisSection.isTable())
-                    .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
-
-        }
+            return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);
+        }).collect(Collectors.toList());
+    }
 
 
-        Set<FileAttribute> addedFileAttributes = new HashSet<>();
-        Set<Entity> entities = new HashSet<>();
-        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
+    private SectionSearchableTextPair toSectionSearchableTextPair(Dictionary dictionary,
+                                                                  AnalyzeRequest analyzeRequest,
+                                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                                                  SectionText reanalysisSection,
+                                                                  Entities entities) {
 
-            if(!addedFileAttributes.isEmpty()) {
-                //Section.Builder provides immutable list.
-                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
-                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
-                mergedFileAttributes.addAll(addedFileAttributes);
-                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
-            }
-
-            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
-
-            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
-
-            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
-
-            var entriesWithoutSurroundingText = analysedSection.getEntities()
-                    .stream()
-                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
-                    .collect(Collectors.toSet());
-
-            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                        sectionSearchableTextPair.getSearchableText(),
-                        dictionary,
-                        sectionSearchableTextPair.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
-            }
-
-            entities.addAll(analysedSection.getEntities());
-
-            if (!local) {
-                for (Image image : analysedSection.getImages()) {
-                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
-                }
-                addLocalValuesToDictionary(analysedSection, dictionary);
-            }
-
-        });
-
-        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
+        return new SectionSearchableTextPair(Section.builder()
+                .isLocal(false)
+                .dictionaryTypes(dictionary.getTypes())
+                .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities().stream(),
+                        hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
+                .nerEntities(entities.getNerEntities())
+                .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
+                .searchText(reanalysisSection.getSearchableText().toString())
+                .headline(reanalysisSection.getHeadline())
+                .sectionNumber(reanalysisSection.getSectionNumber())
+                .tabularData(reanalysisSection.getTabularData())
+                .searchableText(reanalysisSection.getSearchableText())
+                .dictionary(dictionary)
+                .images(reanalysisSection.getImages())
+                .sectionAreas(reanalysisSection.getSectionAreas())
+                .fileAttributes(analyzeRequest.getFileAttributes())
+                .manualRedactions(analyzeRequest.getManualRedactions())
+                .isInTable(reanalysisSection.isTable())
+                .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts());
     }
 
 
@@ -244,7 +285,7 @@ public class EntityRedactionService {
     private Map<Integer, Set<Entity>> getHintsPerSection(Set<Entity> entities, Dictionary dictionary) {
 
         Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
-        entities.stream().forEach(entity -> {
+        entities.forEach(entity -> {
             if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
                 hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity);
             }
@@ -269,95 +310,4 @@ public class EntityRedactionService {
         }));
     }
 
-
-    @Timed("redactmanager_findEntities")
-    private Entities findEntities(SearchableText searchableText,
-                                  String headline,
-                                  int sectionNumber,
-                                  Dictionary dictionary,
-                                  boolean local,
-                                  NerEntities nerEntities,
-                                  List<Integer> cellStarts,
-                                  ManualRedactions manualRedactions) {
-
-        Set<Entity> found = new HashSet<>();
-        String searchableString = searchableText.asString();
-
-        if (StringUtils.isEmpty(searchableString)) {
-            return new Entities(new HashSet<>(), new HashSet<>());
-        }
-
-        String lowercaseInputString = searchableString.toLowerCase();
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
-
-            var searchImplementation = local ? model.getLocalSearch() : model.getEntriesSearch();
-            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
-                    searchImplementation,
-                    model,
-                    new FindEntityDetails(model.getType(),
-                            headline,
-                            sectionNumber,
-                            !local,
-                            model.isDossierDictionary(),
-                            local ? Engine.RULE : Engine.DICTIONARY,
-                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
-
-            var withPositions = EntitySearchUtils.clearAndFindPositions(entities, searchableText, dictionary, manualRedactions);
-
-            EntitySearchUtils.addOrAddEngine(found, withPositions.stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet()));
-        }
-
-        Set<Entity> nerFound = new HashSet<>();
-        if (!local) {
-            nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline));
-        }
-
-        EntitySearchUtils.removeEntitiesContainedInLarger(found);
-        return new Entities(found, nerFound);
-    }
-
-
-    private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities, List<Integer> cellStarts, String headline) {
-
-        Set<Entity> entities = new HashSet<>();
-
-        if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) {
-            nerEntities.getData().get(sectionNumber).forEach(res -> {
-                if (cellStarts == null || cellStarts.isEmpty()) {
-                    entities.add(new Entity(res.getValue(),
-                            res.getType(),
-                            res.getStartOffset(),
-                            res.getEndOffset(),
-                            headline,
-                            sectionNumber,
-                            false,
-                            false,
-                            Engine.NER,
-                            EntityType.RECOMMENDATION));
-                } else {
-                    boolean intersectsCellStart = false;
-                    for (Integer cellStart : cellStarts) {
-                        if (res.getStartOffset() < cellStart && cellStart < res.getEndOffset()) {
-                            intersectsCellStart = true;
-                            break;
-                        }
-                    }
-                    if (!intersectsCellStart) {
-                        entities.add(new Entity(res.getValue(),
-                                res.getType(),
-                                res.getStartOffset(),
-                                res.getEndOffset(),
-                                headline,
-                                sectionNumber,
-                                false,
-                                false,
-                                Engine.NER,
-                                EntityType.RECOMMENDATION));
-                    }
-                }
-            });
-        }
-        return entities;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -11,8 +11,8 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
@@ -273,14 +273,14 @@ public final class EntitySearchUtils {
                 existing.setLegalBasis(found.getLegalBasis());
                 existing.setMatchedRule(found.getMatchedRule());
                 existing.setRedactionReason(found.getRedactionReason());
-                if (isOneARecommendationAndTheOtherEntity(found, existing)) {
+                if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType()
+                        .equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
                     existing.setEntityType(EntityType.ENTITY);
                     if (found.isRedaction()) {
                         existing.setRedaction(true);
                     }
                 }
-            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType()) &&
-                    !(isOneARecommendationAndTheOtherEntity(found, existing) && existing.isRedaction() && found.isRedaction()) ) {
+            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
                 entities.remove(found);
                 entities.add(found);
             }
@@ -289,13 +289,6 @@ public final class EntitySearchUtils {
         }
     }
 
-    private boolean isOneARecommendationAndTheOtherEntity(Entity entityOne, Entity entityTwo) {
-
-        var entityTypeOne = entityOne.getEntityType();
-        var entityTypeTwo = entityTwo.getEntityType();
-        return entityTypeTwo.equals(EntityType.RECOMMENDATION) && entityTypeOne.equals(EntityType.ENTITY)
-                || entityTypeTwo.equals(EntityType.ENTITY) && entityTypeOne.equals(EntityType.RECOMMENDATION);
-    }
 
     public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found) {
         // HashSet keeps old value but we want the new.

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/FindEntityDetails.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java

@@ -9,7 +9,7 @@ import java.util.Map;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -53,15 +53,14 @@ public class PdfSegmentationService {
     public Document parseDocument(String dossierId, String fileId, InputStream documentInputStream, Map<Integer, List<PdfImage>> pdfImages) throws IOException {
 
         PDDocument pdDocument = null;
+        File tempFile = null;
         try {
-            //create tempFile
-            File tempFile = FileUtils.createTempFile("document", ".pdf");
-
             Map<Integer, List<PdfTableCell>> pdfTableCells = new HashMap<>();
             if (redactionServiceSettings.isCvTableParsingEnabled()) {
                 pdfTableCells = tableService.convertTables(dossierId, fileId);
             }
 
+            tempFile = FileUtils.createTempFile("document", ".pdf");
             try (var fos = new FileOutputStream(tempFile)) {
                 IOUtils.copy(documentInputStream, fos);
 
@@ -74,44 +73,7 @@ public class PdfSegmentationService {
                 long pageCount = pdDocument.getNumberOfPages();
 
                 for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
-
-                    PDFLinesTextStripper stripper = new PDFLinesTextStripper();
-                    PDPage pdPage = pdDocument.getPage(pageNumber - 1);
-                    stripper.setPageNumber(pageNumber);
-                    stripper.setStartPage(pageNumber);
-                    stripper.setEndPage(pageNumber);
-                    stripper.setPdpage(pdPage);
-                    stripper.getText(pdDocument);
-
-                    PDRectangle pdr = pdPage.getMediaBox();
-
-                    int rotation = pdPage.getRotation();
-                    boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-                    PDRectangle cropbox = pdPage.getCropBox();
-                    CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
-                            stripper.getRulings(),
-                            stripper.getMinCharWidth(),
-                            stripper.getMaxCharHeight());
-                    Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
-
-                    page.setRotation(rotation);
-                    page.setLandscape(isLandscape);
-                    page.setPageNumber(pageNumber);
-                    page.setPageWidth(cropbox.getWidth());
-                    page.setPageHeight(cropbox.getHeight());
-
-                    // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
-                    if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
-                        page.setImages(pdfImages.get(pageNumber));
-                        imageService.findOcr(page);
-                    }
-
-                    tableExtractionService.extractTables(cleanRulings, page);
-                    buildPageStatistics(page);
-                    increaseDocumentStatistics(page, document);
-
-                    pages.add(page);
+                    processPage(pdfImages, pdDocument, pdfTableCells, document, pages, pageNumber);
                 }
 
                 document.setPages(pages);
@@ -120,22 +82,65 @@ public class PdfSegmentationService {
                 sectionsBuilderService.buildSections(document);
                 sectionsBuilderService.addImagesToSections(document);
 
-                IOUtils.close(pdDocument);
-
-                if (!tempFile.delete()) {
-                    log.warn("Could not delete tmp file");
-                }
-
                 return document;
             }
         } finally {
             if (pdDocument != null) {
                 pdDocument.close();
             }
+
+            FileUtils.deleteFile(tempFile);
         }
     }
 
 
+    private void processPage(Map<Integer, List<PdfImage>> pdfImages,
+                           PDDocument pdDocument,
+                           Map<Integer, List<PdfTableCell>> pdfTableCells,
+                           Document document,
+                           List<Page> pages,
+                           int pageNumber) throws IOException {
+
+        PDFLinesTextStripper stripper = new PDFLinesTextStripper();
+        PDPage pdPage = pdDocument.getPage(pageNumber - 1);
+        stripper.setPageNumber(pageNumber);
+        stripper.setStartPage(pageNumber);
+        stripper.setEndPage(pageNumber);
+        stripper.setPdpage(pdPage);
+        stripper.getText(pdDocument);
+
+        PDRectangle pdr = pdPage.getMediaBox();
+
+        int rotation = pdPage.getRotation();
+        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+        PDRectangle cropbox = pdPage.getCropBox();
+        CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                stripper.getRulings(),
+                stripper.getMinCharWidth(),
+                stripper.getMaxCharHeight());
+        Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
+
+        page.setRotation(rotation);
+        page.setLandscape(isLandscape);
+        page.setPageNumber(pageNumber);
+        page.setPageWidth(cropbox.getWidth());
+        page.setPageHeight(cropbox.getHeight());
+
+        // If images is ocr needs to be calculated before textBlocks are moved into tables, otherwise findOcr algorithm needs to be adopted.
+        if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
+            page.setImages(pdfImages.get(pageNumber));
+            imageService.findOcr(page);
+        }
+
+        tableExtractionService.extractTables(cleanRulings, page);
+        buildPageStatistics(page);
+        increaseDocumentStatistics(page, document);
+
+        pages.add(page);
+    }
+
+
     private void increaseDocumentStatistics(Page page, Document document) {
 
         if (!page.isLandscape()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TableService.java

@@ -9,7 +9,7 @@ import java.util.Map;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
 import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableCells;
 import com.iqser.red.service.redaction.v1.server.redaction.model.table.TableServiceResponse;
@@ -60,4 +60,4 @@ public class TableService {
         return pdfTableCells;
     }
 
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -4,13 +4,14 @@ import java.io.InputStream;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.ImportedRedactions;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 
@@ -31,14 +32,14 @@ public class RedactionStorageService {
     @SneakyThrows
     public InputStream getStoredObject(String storageId) {
 
-        return storageService.getObject(storageId).getInputStream();
+        return storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream();
     }
 
 
     @SneakyThrows
     public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) {
 
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
+        storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream);
     }
 
 
@@ -46,7 +47,7 @@ public class RedactionStorageService {
     @Timed("redactmanager_storeObject")
     public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
 
-        storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
+        storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
     }
 
 
@@ -54,7 +55,7 @@ public class RedactionStorageService {
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Imported redactions not available.");
             return null;
@@ -66,7 +67,7 @@ public class RedactionStorageService {
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
@@ -79,7 +80,7 @@ public class RedactionStorageService {
     public Text getText(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Text not available.");
             return null;
@@ -91,7 +92,7 @@ public class RedactionStorageService {
     public NerEntities getNerEntities(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("NER Entities are not available.");
         }
@@ -102,7 +103,7 @@ public class RedactionStorageService {
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
         } catch (StorageObjectDoesNotExist e) {
             throw new NotFoundException("Section Grid is not available.");
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;
 
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java

@@ -25,8 +25,6 @@ public class Cell extends Rectangle {
 
     private boolean isHeaderCell;
 
-    private static final int MIN_SIZE = 1;
-
 
     public Cell(Point2D topLeft, Point2D bottomRight) {
 
@@ -68,10 +66,4 @@ public class Cell extends Rectangle {
         return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
     }
 
-
-    public boolean hasMinimumSize() {
-
-        return this.getHeight() >= MIN_SIZE && this.getWidth() >= MIN_SIZE;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java

@@ -261,9 +261,7 @@ public class Table extends AbstractTextContainer {
                     if (intersectionCell.isPresent()) {
                         cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
                     }
-                    if (cell.hasMinimumSize()) {
-                        row.add(cell);
-                    }
+                    row.add(cell);
                 }
                 prevX = x;
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java

@@ -88,7 +88,7 @@ public class TableExtractionService {
         for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
             TextBlock textBlock = (TextBlock) abstractTextContainer;
             for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
+                if (cell.intersects(textBlock.getPdfMinX(),
                         textBlock.getPdfMinY(),
                         textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                         textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -109,7 +109,7 @@ public class TableExtractionService {
 
             List<Cell> overlappingCells = new ArrayList<>();
             for (Cell c : cells) {
-                if (c.hasMinimumSize() && c.intersects(area)) {
+                if (c.intersects(area)) {
                     overlappingCells.add(c);
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/utils/FileUtils.java

@@ -20,19 +20,31 @@ public class FileUtils {
     }
 
 
+    /**
+     * Deletes a file; logs a message with the reason if the deletion fails.
+     * This method is null-safe.
+     * @param file The file to delete.  Can be null.
+     */
+    public void deleteFile(File file) {
+        if (file != null) {
+            try {
+                Files.deleteIfExists(file.toPath());
+            } catch (IOException ex) {
+                log.warn("Could not delete file!", ex);
+            }
+        }
+    }
+
+
     // We don't need to check the results of the permission setters below,
     // since we're manipulating a file we created ourselves.
-    @SuppressWarnings("ResultOfMethodCallIgnored")
+    @SuppressWarnings({"ResultOfMethodCallIgnored", "squid:S899"})
     private void setRWPermissionsOnlyForOwner(File tempFile) {
 
         try {
-            // deny for all
-            tempFile.setReadable(false);
-            tempFile.setWritable(false);
-            tempFile.setExecutable(false);
-            // allow for owner
             tempFile.setReadable(true, true);
             tempFile.setWritable(true, true);
+            tempFile.setExecutable(false);
         } catch (SecurityException ex) {
             // This should never happen since we're creating a temp file ourselves.
             log.warn("Caught an exception during temp file creation.  This should not happend.  Check the code.", ex);

redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml

@@ -1,11 +1,17 @@
 server:
   port: 8083
 
-configuration-service.url: "http://localhost:8081"
-file-management-service.url: "http://localhost:8085"
+persistence-service.url: "http://localhost:8085"
 
 storage:
   bucket-name: 'redaction'
   endpoint: 'http://localhost:9000'
   key: minioadmin
   secret: minioadmin
+
+
+redaction-service:
+  enableImageClassification: false
+  cvTableParsingEnabled: false
+  nerServiceEnabled: false
+  priorityMode: false

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -38,8 +38,4 @@ management:
 
 
 storage:
-  signer-type: 'AWSS3V4SignerType'
-  bucket-name: 'redaction'
-  region: 'us-east-1'
-  endpoint: 'https://s3.amazonaws.com'
   backend: 's3'

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
 
-import org.junit.Test;
-import org.junit.runner.RunWith;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -21,16 +21,19 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
 
-import static org.mockito.ArgumentMatchers.*;
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class DictionaryServiceTest {
@@ -61,6 +64,8 @@ public class DictionaryServiceTest {
     @Test
     public void testDictionaryServiceConsistency() {
 
+        TenantContext.setTenantId("redaction");
+
         when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
         when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId",
                 "#cccccc",
@@ -73,6 +78,7 @@ public class DictionaryServiceTest {
                 "#cccccc",
                 "#cccccc",
                 "#cccccc",
+                "#cccccc",
                 "#cccccc"));
 
         var type1 = new Type();