RED-6619 - change pom to fforesight again

Merge in RED/redaction-service from RED-6619_1 to master

* commit 'aff7074b4012627a4e8b64b2f4a779b18feed50a':
  RED-6619 - renamed variables
  RED-6619 - reformat code
  RED-6619 - delete unnecessary import
  RED-6619 - moved not the logic to a boolean, but the 1 into a constant
  RED-6619 - fix integration-tests by adding versions and move the hasMinimumSize-logic into own boolea
  RED-6619 - added missing ' (typo)
  RED-6619 - add logic to ignore found table-cells with height or width < 1. Also: Fix the tests and add new segmentation-tests and 1 redaction-integration-test. Renamed the latter to fit maven regexp
  RED-6619 - add tests for table-extraction

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -103,7 +103,7 @@ public class PlanSpec {
                                         .enabled(true),
                                 new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL),
                                 new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.knecon.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                 .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                 .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)
@@ -128,7 +128,7 @@ public class PlanSpec {
                                 createJUnitParserTask().description("Resultparser")
                                         .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
                                         .enabled(true))
-                        .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
+                        .dockerConfiguration(new DockerConfiguration().image("nexus.knecon.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                 .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                 .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RED / " + SERVICE_NAME)

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -54,7 +54,7 @@ mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
     -Dmaven.wagon.http.ssl.insecure=true \
     -Dmaven.wagon.http.ssl.allowall=true \
     -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+    -DaltDeploymentRepository=iqser_release::default::https://nexus.knecon.com/repository/red-platform-releases
 
 mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \

bamboo-specs/src/main/resources/scripts/sonar-java.sh

@@ -22,7 +22,7 @@ then
         -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
         sonar:sonar \
         -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.host.url=https://sonarqube.knecon.com \
         -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
         -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
         -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
@@ -34,7 +34,7 @@ else
         -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
         sonar:sonar \
         -Dsonar.projectKey=RED_$SERVICE_NAME \
-        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.host.url=https://sonarqube.knecon.com \
         -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
         -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
         -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \

redaction-service-image-v1/pom.xml

@@ -3,9 +3,9 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <groupId>com.iqser.red</groupId>
+        <groupId>com.knecon.fforesight</groupId>
         <artifactId>platform-docker-dependency</artifactId>
-        <version>1.2.0</version>
+        <version>0.1.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>

redaction-service-v1/pom.xml

@@ -5,14 +5,14 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.17.0</version>
+        <version>2.2.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>redaction-service-v1</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>1.0-SNAPSHOT</version>
+    <version>4.0-SNAPSHOT</version>
 
     <packaging>pom</packaging>
 
@@ -23,7 +23,6 @@
 
     <properties>
         <pdfbox.version>2.0.24</pdfbox.version>
-        <dsljson.version>1.9.9</dsljson.version>
     </properties>
 
 
@@ -32,7 +31,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.22.0</version>
+                <version>2.1.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <artifactId>redaction-service-v1</artifactId>
         <groupId>com.iqser.red.service</groupId>
-        <version>1.0-SNAPSHOT</version>
+        <version>4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>redaction-service-api-v1</artifactId>
@@ -17,13 +17,6 @@
 
     <dependencies>
 
-        <!-- https://mvnrepository.com/artifact/com.dslplatform/dsl-json-java8 -->
-        <dependency>
-            <groupId>com.dslplatform</groupId>
-            <artifactId>dsl-json-java8</artifactId>
-            <version>${dsljson.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-web</artifactId>
@@ -55,10 +48,13 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <annotationProcessors>
-                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
-                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
-                    </annotationProcessors>
+                    <annotationProcessorPaths>
+                        <path>
+                            <groupId>org.projectlombok</groupId>
+                            <artifactId>lombok</artifactId>
+                            <version>${lombok.version}</version>
+                        </path>
+                    </annotationProcessorPaths>
                 </configuration>
             </plugin>
         </plugins>

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <artifactId>redaction-service-v1</artifactId>
         <groupId>com.iqser.red.service</groupId>
-        <version>1.0-SNAPSHOT</version>
+        <version>4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>redaction-service-server-v1</artifactId>
@@ -14,10 +14,10 @@
     <properties>
         <drools.version>7.73.0.Final</drools.version>
         <kie.version>7.73.0.Final</kie.version>
-        <locationtech.version>1.18.2</locationtech.version>
-        <javaassist.version>3.28.0-GA</javaassist.version>
+        <locationtech.version>1.19.0</locationtech.version>
+        <javaassist.version>3.29.2-GA</javaassist.version>
         <ahocorasick.version>0.6.3</ahocorasick.version>
-        <jackson.version>2.13.2</jackson.version>
+        <jackson.version>2.14.2</jackson.version>
     </properties>
 
     <dependencies>
@@ -30,6 +30,7 @@
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>storage-commons</artifactId>
+            <version>2.1.0</version>
         </dependency>
 
         <dependency>
@@ -134,10 +135,13 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <annotationProcessors>
-                        <annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
-                        <annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
-                    </annotationProcessors>
+                    <annotationProcessorPaths>
+                        <path>
+                            <groupId>org.projectlombok</groupId>
+                            <artifactId>lombok</artifactId>
+                            <version>${lombok.version}</version>
+                        </path>
+                    </annotationProcessorPaths>
                 </configuration>
             </plugin>
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -9,17 +9,17 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
-import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig;
 import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
 import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyWebConfiguration;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
 
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
-@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class})
+@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class, MetricsConfiguration.class, StorageAutoConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/MetricsConfiguration.java

@@ -0,0 +1,26 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+import com.iqser.gin4.commons.metrics.MetricCommonsConfiguration;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerFactory;
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
+
+@Configuration
+@Import({MetricCommonsConfiguration.class})
+public class MetricsConfiguration {
+
+    private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
+
+
+    @Bean
+    @Qualifier(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME)
+    public FunctionTimerValues redactmanagerAnalyzePagewise(FunctionTimerFactory factory) {
+
+        return factory.create(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Footer.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
@@ -17,7 +16,6 @@ public class Footer {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public SearchableText getSearchableText() {
 
         SearchableText searchableText = new SearchableText();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Header.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
@@ -17,7 +16,6 @@ public class Header {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public SearchableText getSearchableText() {
 
         SearchableText searchableText = new SearchableText();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -1,7 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
@@ -13,11 +18,8 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.*;
-
 @Data
 @Builder
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class SectionText {
@@ -48,7 +50,6 @@ public class SectionText {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public SearchableText getSearchableText() {
 
         SearchableText searchableText = new SearchableText();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -9,7 +8,6 @@ import lombok.NoArgsConstructor;
 
 @Data
 @Builder
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class SimplifiedSectionText {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java

@@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.dslplatform.json.CompiledJson;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -12,7 +10,6 @@ import lombok.NoArgsConstructor;
 
 @Data
 @Builder
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class SimplifiedText {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Text.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -10,7 +9,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 @Data
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class Text {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -19,7 +17,6 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 @Builder
 @Data
-@CompiledJson
 @NoArgsConstructor
 public class TextBlock extends AbstractTextContainer {
 
@@ -52,7 +49,6 @@ public class TextBlock extends AbstractTextContainer {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public TextDirection getDir() {
 
         return sequences.get(0).getDir();
@@ -60,7 +56,6 @@ public class TextBlock extends AbstractTextContainer {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     private float getPageHeight() {
 
         return sequences.get(0).getPageHeight();
@@ -68,7 +63,6 @@ public class TextBlock extends AbstractTextContainer {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     private float getPageWidth() {
 
         return sequences.get(0).getPageWidth();
@@ -86,7 +80,6 @@ public class TextBlock extends AbstractTextContainer {
      * @return the minX value in pdf coordinate system
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getPdfMinX() {
 
         if (getDir().getDegrees() == 90) {
@@ -113,7 +106,6 @@ public class TextBlock extends AbstractTextContainer {
      * @return the maxX value in pdf coordinate system
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getPdfMaxX() {
 
         if (getDir().getDegrees() == 90) {
@@ -140,7 +132,6 @@ public class TextBlock extends AbstractTextContainer {
      * @return the minY value in pdf coordinate system
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getPdfMinY() {
 
         if (getDir().getDegrees() == 90) {
@@ -168,7 +159,6 @@ public class TextBlock extends AbstractTextContainer {
      * @return the maxY value in pdf coordinate system
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getPdfMaxY() {
 
         if (getDir().getDegrees() == 90) {
@@ -288,7 +278,6 @@ public class TextBlock extends AbstractTextContainer {
 
     @Override
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public String getText() {
 
         StringBuilder sb = new StringBuilder();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/UnclassifiedText.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
@@ -17,7 +16,6 @@ public class UnclassifiedText {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public SearchableText getSearchableText() {
 
         SearchableText searchableText = new SearchableText();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.client.model;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -9,7 +8,6 @@ import lombok.NoArgsConstructor;
 
 @Data
 @Builder
-@CompiledJson
 @AllArgsConstructor
 @NoArgsConstructor
 public class EntityRecogintionEntity {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java

@@ -4,14 +4,12 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
 @Data
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class NerEntities {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java

@@ -6,7 +6,6 @@ import java.security.SecureRandom;
 import java.security.spec.KeySpec;
 import java.util.Base64;
 
-import javax.annotation.PostConstruct;
 import javax.crypto.Cipher;
 import javax.crypto.SecretKey;
 import javax.crypto.SecretKeyFactory;
@@ -17,6 +16,7 @@ import javax.crypto.spec.SecretKeySpec;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import jakarta.annotation.PostConstruct;
 import lombok.SneakyThrows;
 
 @Service

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java

@@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.parsing.model;
 import org.apache.pdfbox.text.TextPosition;
 import org.springframework.beans.BeanUtils;
 
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
 import lombok.AllArgsConstructor;
@@ -17,7 +15,6 @@ import lombok.SneakyThrows;
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@CompiledJson
 public class RedTextPosition {
 
     private String textMatrix;
@@ -39,17 +36,14 @@ public class RedTextPosition {
 
     // not used in reanalysis
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     private float widthOfSpace;
 
     // not used in reanalysis
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     private float fontSizeInPt;
 
     // not used in reanalysis
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     private String fontName;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextDirection.java

@@ -35,13 +35,6 @@ public enum TextDirection {
     }
 
 
-    @com.dslplatform.json.JsonValue
-    public float jsonValue() {
-
-        return getDegrees();
-    }
-
-
     @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
     public static TextDirection fromDegrees(float degrees) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -8,8 +8,6 @@ import java.util.stream.Collectors;
 
 import org.apache.pdfbox.text.TextPosition;
 
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
@@ -25,7 +23,6 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Data
 @Builder
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 @JsonIgnoreProperties({"empty"})
@@ -144,7 +141,6 @@ public class TextPositionSequence implements CharSequence {
      * @return the text direction adjusted minX value
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getMinXDirAdj() {
 
         return textPositions.get(0).getXDirAdj();
@@ -159,7 +155,6 @@ public class TextPositionSequence implements CharSequence {
      * @return the text direction adjusted maxX value
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getMaxXDirAdj() {
 
         return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
@@ -174,7 +169,6 @@ public class TextPositionSequence implements CharSequence {
      * @return the text direction adjusted minY value. The upper border of the bounding box of the word.
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getMinYDirAdj() {
 
         return textPositions.get(0).getYDirAdj() - getTextHeight();
@@ -189,7 +183,6 @@ public class TextPositionSequence implements CharSequence {
      * @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getMaxYDirAdj() {
 
         return textPositions.get(0).getYDirAdj();
@@ -198,7 +191,6 @@ public class TextPositionSequence implements CharSequence {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getTextHeight() {
 
         return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
@@ -206,7 +198,6 @@ public class TextPositionSequence implements CharSequence {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getHeight() {
 
         return getMaxYDirAdj() - getMinYDirAdj();
@@ -214,7 +205,6 @@ public class TextPositionSequence implements CharSequence {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getWidth() {
 
         return getMaxXDirAdj() - getMinXDirAdj();
@@ -222,7 +212,6 @@ public class TextPositionSequence implements CharSequence {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public String getFont() {
 
         return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
@@ -230,7 +219,6 @@ public class TextPositionSequence implements CharSequence {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public String getFontStyle() {
 
         String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
@@ -249,7 +237,6 @@ public class TextPositionSequence implements CharSequence {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getFontSize() {
 
         return textPositions.get(0).getFontSizeInPt();
@@ -257,7 +244,6 @@ public class TextPositionSequence implements CharSequence {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getSpaceWidth() {
 
         return textPositions.get(0).getWidthOfSpace();
@@ -274,7 +260,6 @@ public class TextPositionSequence implements CharSequence {
      * @return bounding box of the word in Pdf Coordinate System
      */
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     @SneakyThrows
     public Rectangle getRectangle() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -9,7 +8,6 @@ import lombok.NoArgsConstructor;
 
 @Data
 @Builder
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class Image implements ReasonHolder {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
 import lombok.AllArgsConstructor;
@@ -11,7 +9,6 @@ import lombok.NoArgsConstructor;
 
 @Data
 @Builder
-@CompiledJson
 @NoArgsConstructor
 @AllArgsConstructor
 public class RedRectangle2D {
@@ -25,7 +22,6 @@ public class RedRectangle2D {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public boolean isEmpty() {
 
         return width <= 0.0f || height <= 0.0f;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
@@ -18,7 +17,6 @@ import lombok.Getter;
 public class SearchableText {
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     private transient String stringRepresentation;
     @Getter
     private final List<TextPositionSequence> sequences = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class TenantDictionary {
+
+    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
+    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantRules.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class TenantRules {
+
+    private Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Classification.java

@@ -3,12 +3,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 import java.util.HashMap;
 import java.util.Map;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class Classification {
 
     private Map<String, Float> probabilities = new HashMap<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/FilterGeometry.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class FilterGeometry {
 
     private ImageSize imageSize;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Filters.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class Filters {
 
     private FilterGeometry geometry;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Geometry.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class Geometry {
 
     private float width;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageFormat.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class ImageFormat {
 
     private float quotient;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageMetadata.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class ImageMetadata {
 
     private Classification classification;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
@@ -11,7 +9,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 @Data
-@CompiledJson
 public class ImageServiceResponse {
 
     private String dossierId;
@@ -19,7 +16,6 @@ public class ImageServiceResponse {
 
     @JsonProperty(value = "imageMetadata")
     @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
     private List<ImageMetadata> data = new ArrayList<>();
 
     private List<ImageMetadata> dataCV = new ArrayList<>();
@@ -27,7 +23,6 @@ public class ImageServiceResponse {
 
     @JsonProperty(value = "imageMetadata")
     @JsonAlias("data")
-    @JsonAttribute(alternativeNames = {"imageMetadata"})
     public void setData(List<ImageMetadata> data) {this.data = data;}
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageSize.java

@@ -1,11 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
-
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class ImageSize {
 
     private float quotient;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Position.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class Position {
 
     private float x1;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Probability.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class Probability {
 
     private boolean unconfident;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/PageInfo.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.table;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class PageInfo {
 
     private int number;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/TableCells.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model.table;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class TableCells {
 
     private float x0;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/TableData.java

@@ -3,12 +3,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.table;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class TableData {
 
     private PageInfo pageInfo;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/table/TableServiceResponse.java

@@ -3,12 +3,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.table;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.dslplatform.json.CompiledJson;
 
 import lombok.Data;
 
 @Data
-@CompiledJson
 public class TableServiceResponse {
 
     private String dossierId;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,24 +1,43 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-
-import feign.FeignException;
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.Color;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;
 
-import java.awt.*;
-import java.util.List;
-import java.util.*;
-import java.util.stream.Collectors;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import feign.FeignException;
+import io.micrometer.core.annotation.Timed;
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
@@ -28,22 +47,39 @@ public class DictionaryService {
     private static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
-    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
-    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+    private final RedactionServiceSettings settings;
+
+    private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
 
 
+    @PostConstruct
+    protected void createCache() {
+
+        tenantDictionaryCache = CacheBuilder.newBuilder()
+                .maximumSize(settings.getDictionaryCacheMaximumSize())
+                .expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS)
+                .build(new CacheLoader<>() {
+                    public TenantDictionary load(String key) {
+
+                        return new TenantDictionary();
+                    }
+                });
+    }
+
+
+    @SneakyThrows
     @Timed("redactmanager_updateDictionary")
     public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
 
         log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
         long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
-        var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId);
         if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
         }
 
         long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
-        var dossierDictionary = dictionariesByDossier.get(dossierId);
+        var dossierDictionary = getDossierDictionary(dossierId);
         if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
         }
@@ -52,13 +88,14 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDictionaryIncrements")
     public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
 
         DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
 
         Set<DictionaryIncrementValue> newValues = new HashSet<>();
-        List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
+        List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary();
         dictionaryModels.forEach(dictionaryModel -> {
             dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                 if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@@ -77,8 +114,8 @@ public class DictionaryService {
             });
         });
 
-        if (dictionariesByDossier.containsKey(dossierId)) {
-            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
+        if (dossierDictionaryExists(dossierId)) {
+            dictionaryModels = getDossierDictionary(dossierId).getDictionary();
             dictionaryModels.forEach(dictionaryModel -> {
                 dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                     if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
@@ -102,6 +139,7 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {
 
         try {
@@ -115,10 +153,10 @@ public class DictionaryService {
 
                     Optional<DictionaryModel> oldModel;
                     if (dossierId == null) {
-                        var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
+                        var representation = getDossierTemplateDictionary(dossierTemplateId);
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     } else {
-                        var representation = dictionariesByDossier.get(dossierId);
+                        var representation = getDossierDictionary(dossierId);
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     }
 
@@ -178,9 +216,9 @@ public class DictionaryService {
                 dictionaryRepresentation.setDictionary(dictionary);
 
                 if (dossierId == null) {
-                    dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
+                    addDictionaryRepresentationForDossierTemplate(dossierTemplateId, dictionaryRepresentation);
                 } else {
-                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
+                    addDictionaryRepresentationForDossier(dossierId, dictionaryRepresentation);
                 }
             }
         } catch (FeignException e) {
@@ -219,19 +257,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getColor(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.getColor();
         }
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
+        return getDossierTemplateDictionary(dossierTemplateId).getDefaultColor();
     }
 
 
+    @SneakyThrows
     public boolean isHint(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.isHint();
         }
@@ -239,20 +279,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDeepCopyDictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
         List<DictionaryModel> copy = new ArrayList<>();
 
-        var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
         dossierTemplateRepresentation.getDictionary().forEach(dm -> {
             copy.add(SerializationUtils.clone(dm));
         });
 
         //TODO merge dictionaries if they have same names
         long dossierDictionaryVersion = -1;
-        if (dictionariesByDossier.containsKey(dossierId)) {
-            var dossierRepresentation = dictionariesByDossier.get(dossierId);
+        if (dossierDictionaryExists(dossierId)) {
+            var dossierRepresentation = getDossierDictionary(dossierId);
             dossierRepresentation.getDictionary().forEach(dm -> {
                 copy.add(SerializationUtils.clone(dm));
             });
@@ -264,9 +305,45 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getNotRedactedColor(String dossierTemplateId) {
 
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
+        return getDossierTemplateDictionary(dossierTemplateId).getNotRedactedColor();
+    }
+
+
+    @SneakyThrows
+    private void addDictionaryRepresentationForDossierTemplate(String dossierTemplateId, DictionaryRepresentation dictionaryRepresentation) {
+
+        tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation);
+    }
+
+
+    @SneakyThrows
+    private void addDictionaryRepresentationForDossier(String dossierId, DictionaryRepresentation dictionaryRepresentation) {
+
+        tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation);
+    }
+
+
+    @SneakyThrows
+    private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) {
+
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
+    }
+
+
+    @SneakyThrows
+    private DictionaryRepresentation getDossierDictionary(String dossierId) {
+
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
+    }
+
+
+    @SneakyThrows
+    private boolean dossierDictionaryExists(String dossierId) {
+
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -1,11 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
@@ -16,11 +16,20 @@ import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
 import org.springframework.stereotype.Service;
 
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import com.iqser.red.service.redaction.v1.server.redaction.model.TenantRules;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import io.micrometer.core.annotation.Timed;
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 
 @Service
 @RequiredArgsConstructor
@@ -30,7 +39,24 @@ public class DroolsExecutionService {
 
     private final Map<String, KieContainer> kieContainers = new HashMap<>();
 
-    private final Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
+    private final RedactionServiceSettings settings;
+
+    private LoadingCache<String, TenantRules> tenantRulesCache;
+
+
+    @PostConstruct
+    protected void createCache() {
+
+        tenantRulesCache = CacheBuilder.newBuilder()
+                .maximumSize(settings.getDictionaryCacheMaximumSize())
+                .expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS)
+                .build(new CacheLoader<>() {
+                    public TenantRules load(String key) {
+
+                        return new TenantRules();
+                    }
+                });
+    }
 
 
     public KieContainer getKieContainer(String dossierTemplateId) {
@@ -61,13 +87,13 @@ public class DroolsExecutionService {
     public KieContainer updateRules(String dossierTemplateId) {
 
         long version = rulesClient.getVersion(dossierTemplateId);
-        Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
+        Long rulesVersion = getVersionForDossierTemplate(dossierTemplateId);
         if (rulesVersion == null) {
             rulesVersion = -1L;
         }
 
         if (version > rulesVersion) {
-            rulesVersionPerDossierTemplateId.put(dossierTemplateId, version);
+            setRulesVersionForDossierTemplate(dossierTemplateId, version);
             return createOrUpdateKieContainer(dossierTemplateId);
         }
         return getKieContainer(dossierTemplateId);
@@ -126,11 +152,25 @@ public class DroolsExecutionService {
 
     public long getRulesVersion(String dossierTemplateId) {
 
-        Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
+        Long rulesVersion = getVersionForDossierTemplate(dossierTemplateId);
         if (rulesVersion == null) {
             return -1;
         }
         return rulesVersion;
     }
 
+
+    @SneakyThrows
+    private Long getVersionForDossierTemplate(String dossierTemplateId) {
+
+        return tenantRulesCache.get(TenantContext.getTenantId()).getRulesVersionPerDossierTemplateId().get(dossierTemplateId);
+    }
+
+
+    @SneakyThrows
+    private void setRulesVersionForDossierTemplate(String dossierTemplateId, long version) {
+
+        tenantRulesCache.get(TenantContext.getTenantId()).getRulesVersionPerDossierTemplateId().put(dossierTemplateId, version);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -8,13 +8,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
 
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@@ -53,10 +53,8 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
-import io.micrometer.core.instrument.FunctionTimer;
-import io.micrometer.core.instrument.MeterRegistry;
 import lombok.AccessLevel;
-import lombok.Getter;
+import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
@@ -64,8 +62,11 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Service
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@RequiredArgsConstructor
 public class AnalyzeService {
 
+    private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
+
     DictionaryService dictionaryService;
     DroolsExecutionService droolsExecutionService;
     EntityRedactionService entityRedactionService;
@@ -80,50 +81,8 @@ public class AnalyzeService {
     ImageService imageService;
     ImportedRedactionService importedRedactionService;
     SectionFinder sectionFinder;
-    PagewiseCounter analyzeCounter = new PagewiseCounter();
 
-
-    @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
-    public AnalyzeService(DictionaryService dictionaryService,
-                          DroolsExecutionService droolsExecutionService,
-                          EntityRedactionService entityRedactionService,
-                          RedactionLogCreatorService redactionLogCreatorService,
-                          RedactionStorageService redactionStorageService,
-                          PdfSegmentationService pdfSegmentationService,
-                          RedactionChangeLogService redactionChangeLogService,
-                          LegalBasisClient legalBasisClient,
-                          RedactionServiceSettings redactionServiceSettings,
-                          SectionTextBuilderService sectionTextBuilderService,
-                          SectionGridCreatorService sectionGridCreatorService,
-                          ImageService imageService,
-                          ImportedRedactionService importedRedactionService,
-                          SectionFinder sectionFinder,
-                          MeterRegistry meterRegistry) {
-
-        this.dictionaryService = dictionaryService;
-        this.droolsExecutionService = droolsExecutionService;
-        this.entityRedactionService = entityRedactionService;
-        this.redactionLogCreatorService = redactionLogCreatorService;
-        this.redactionStorageService = redactionStorageService;
-        this.pdfSegmentationService = pdfSegmentationService;
-        this.redactionChangeLogService = redactionChangeLogService;
-        this.legalBasisClient = legalBasisClient;
-        this.redactionServiceSettings = redactionServiceSettings;
-        this.sectionTextBuilderService = sectionTextBuilderService;
-        this.sectionGridCreatorService = sectionGridCreatorService;
-        this.imageService = imageService;
-        this.importedRedactionService = importedRedactionService;
-        this.sectionFinder = sectionFinder;
-
-        // This is just here as an example of how to do a timer with a counter.
-        // For a full implementation this should be moved to a common helper class.
-        FunctionTimer.builder("redactmanager_analyze.pagewise", //
-                        analyzeCounter, //
-                        PagewiseCounter::getTotalPageCount, //
-                        PagewiseCounter::getTotalTimeMillis, //
-                        TimeUnit.MILLISECONDS) //
-                .register(meterRegistry);
-    }
+    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
 
     @Timed("redactmanager_analyzeDocumentStructure")
@@ -296,7 +255,7 @@ public class AnalyzeService {
 
         long duration = System.currentTimeMillis() - startTime;
 
-        analyzeCounter.increase(text.getNumberOfPages(), duration);
+        redactmanagerAnalyzePagewiseValues.increase(text.getNumberOfPages(), duration);
 
         return AnalyzeResult.builder()
                 .dossierId(analyzeRequest.getDossierId())
@@ -347,21 +306,4 @@ public class AnalyzeService {
         return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
     }
 
-
-    @FieldDefaults(level = AccessLevel.PRIVATE)
-    @Getter
-    private static final class PagewiseCounter {
-
-        long totalPageCount;
-        long totalTimeMillis;
-
-
-        public void increase(long pageCount, long durationMillis) {
-
-            totalPageCount += pageCount;
-            totalTimeMillis += durationMillis;
-        }
-
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java

@@ -66,7 +66,7 @@ class EntityFinder {
                             !local,
                             model.isDossierDictionary(),
                             local ? Engine.RULE : Engine.DICTIONARY,
-                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+                            local ? EntityType.RECOMMENDATION : EntityType.ENTITY)).stream().filter(e -> !e.isFalsePositive()).collect(Collectors.toSet());
 
             EntitySearchUtils.addOrAddEngine(found, entities);
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -273,14 +273,14 @@ public final class EntitySearchUtils {
                 existing.setLegalBasis(found.getLegalBasis());
                 existing.setMatchedRule(found.getMatchedRule());
                 existing.setRedactionReason(found.getRedactionReason());
-                if (existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY) || existing.getEntityType()
-                        .equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)) {
+                if (isOneARecommendationAndTheOtherEntity(found, existing)) {
                     existing.setEntityType(EntityType.ENTITY);
                     if (found.isRedaction()) {
                         existing.setRedaction(true);
                     }
                 }
-            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
+            } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType()) &&
+                    !(isOneARecommendationAndTheOtherEntity(found, existing) && existing.isRedaction() && found.isRedaction()) ) {
                 entities.remove(found);
                 entities.add(found);
             }
@@ -289,6 +289,13 @@ public final class EntitySearchUtils {
         }
     }
 
+    private boolean isOneARecommendationAndTheOtherEntity(Entity entityOne, Entity entityTwo) {
+
+        var entityTypeOne = entityOne.getEntityType();
+        var entityTypeTwo = entityTwo.getEntityType();
+        return entityTypeTwo.equals(EntityType.RECOMMENDATION) && entityTypeOne.equals(EntityType.ENTITY)
+                || entityTypeTwo.equals(EntityType.ENTITY) && entityTypeOne.equals(EntityType.RECOMMENDATION);
+    }
 
     public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found) {
         // HashSet keeps old value but we want the new.

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java

@@ -24,4 +24,8 @@ public class RedactionServiceSettings {
 
     private boolean priorityMode;
 
+    private long dictionaryCacheMaximumSize = 100;
+
+    private int dictionaryCacheExpireAfterAccessDays = 3;
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.model;
 
-import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
@@ -55,7 +54,6 @@ public abstract class AbstractTextContainer {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getHeight() {
 
         return maxY - minY;
@@ -63,7 +61,6 @@ public abstract class AbstractTextContainer {
 
 
     @JsonIgnore
-    @JsonAttribute(ignore = true)
     public float getWidth() {
 
         return maxX - minX;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Cell.java

@@ -25,6 +25,8 @@ public class Cell extends Rectangle {
 
     private boolean isHeaderCell;
 
+    private static final int MIN_SIZE = 1;
+
 
     public Cell(Point2D topLeft, Point2D bottomRight) {
 
@@ -66,4 +68,10 @@ public class Cell extends Rectangle {
         return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
     }
 
+
+    public boolean hasMinimumSize() {
+
+        return this.getHeight() >= MIN_SIZE && this.getWidth() >= MIN_SIZE;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java

@@ -261,7 +261,9 @@ public class Table extends AbstractTextContainer {
                     if (intersectionCell.isPresent()) {
                         cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
                     }
-                    row.add(cell);
+                    if (cell.hasMinimumSize()) {
+                        row.add(cell);
+                    }
                 }
                 prevX = x;
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java

@@ -88,7 +88,7 @@ public class TableExtractionService {
         for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
             TextBlock textBlock = (TextBlock) abstractTextContainer;
             for (Cell cell : cells) {
-                if (cell.intersects(textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
                         textBlock.getPdfMinY(),
                         textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                         textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -109,7 +109,7 @@ public class TableExtractionService {
 
             List<Cell> overlappingCells = new ArrayList<>();
             for (Cell c : cells) {
-                if (c.intersects(area)) {
+                if (c.hasMinimumSize() && c.intersects(area)) {
                     overlappingCells.add(c);
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -0,0 +1,462 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.AfterEach;
+import org.mockito.stubbing.Answer;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.core.io.ClassPathResource;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+public abstract class AbstractRedactionIntegrationTest {
+
+    protected static final String VERTEBRATE_INDICATOR = "vertebrate";
+    protected static final String DICTIONARY_ADDRESS = "CBI_address";
+    protected static final String DICTIONARY_AUTHOR = "CBI_author";
+    protected static final String DICTIONARY_SPONSOR = "CBI_sponsor";
+    protected static final String DICTIONARY_PII = "PII";
+    protected static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    protected static final String REDACTION_INDICATOR = "redaction_indicator";
+    protected static final String HINT_ONLY_INDICATOR = "hint_only";
+    protected static final String MUST_REDACT_INDICATOR = "must_redact";
+    protected static final String PUBLISHED_INFORMATION_INDICATOR = "published_information";
+    protected static final String TEST_METHOD_INDICATOR = "test_method";
+    protected static final String PURITY_INDICATOR = "purity";
+    protected static final String IMAGE_INDICATOR = "image";
+    protected static final String LOGO_INDICATOR = "logo";
+    protected static final String SIGNATURE_INDICATOR = "signature";
+    protected static final String FORMULA_INDICATOR = "formula";
+    protected static final String OCR_INDICATOR = "ocr";
+    protected static final String DOSSIER_REDACTIONS_INDICATOR = "dossier_redactions";
+    protected static final String IMPORTED_REDACTION_INDICATOR = "imported_redaction";
+    protected static final String ROTATE_SIMPLE_INDICATOR = "RotateSimple";
+
+    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String FORMULA_TYPE_ID = FORMULA_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String SIGNATURE_TYPE_ID = SIGNATURE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String LOGO_TYPE_ID = LOGO_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String OCR_TYPE_ID = OCR_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String IMAGE_TYPE_ID = IMAGE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String PURITY_TYPE_ID = PURITY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+
+    @Autowired
+    protected RedactionController redactionController;
+
+    @Autowired
+    protected AnnotationService annotationService;
+
+    @Autowired
+    protected AnalyzeService analyzeService;
+
+    @Autowired
+    protected ObjectMapper objectMapper;
+
+    @Autowired
+    protected RedactionStorageService redactionStorageService;
+
+    @Autowired
+    protected StorageService storageService;
+
+    @Autowired
+    protected ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
+
+    @MockBean
+    protected AmazonS3 amazonS3;
+
+    @MockBean
+    protected RabbitTemplate rabbitTemplate;
+
+    @MockBean
+    protected LegalBasisClient legalBasisClient;
+
+    protected final Map<String, List<String>> dictionary = new HashMap<>();
+    protected final Map<String, List<String>> dossierDictionary = new HashMap<>();
+    protected final Map<String, List<String>> falsePositive = new HashMap<>();
+    protected final Map<String, List<String>> falseRecommendation = new HashMap<>();
+    protected final Map<String, String> typeColorMap = new HashMap<>();
+    protected final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    protected final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
+    protected final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
+    protected final Map<String, Integer> rankTypeMap = new HashMap<>();
+    protected final Colors colors = new Colors();
+    protected final Map<String, Long> reanlysisVersions = new HashMap<>();
+    protected final Set<String> deleted = new HashSet<>();
+
+    protected final static String TEST_DOSSIER_ID = "123";
+    protected final static String TEST_FILE_ID = "123";
+
+    @MockBean
+    protected RulesClient rulesClient;
+
+    @MockBean
+    protected DictionaryClient dictionaryClient;
+
+
+    @AfterEach
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    protected void mockDictionaryCalls(Long version) {
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(PII_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(OCR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS_INDICATOR,true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,true));
+
+    }
+
+
+    protected void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(DICTIONARY_AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(DICTIONARY_SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(DICTIONARY_ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>());
+
+        falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+    }
+
+
+    protected void loadOnlyDictionaryForSimpleFile() {
+
+        dictionary.clear();
+        dictionary.computeIfAbsent(ROTATE_SIMPLE_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+    }
+
+
+    @SneakyThrows
+    protected static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        List<String> stringList = Files.readAllLines(new File(resource.getPath()).toPath());
+        return String.join("\n", stringList);
+    }
+
+
+    protected void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATE_INDICATOR, "#ff85f7");
+        typeColorMap.put(DICTIONARY_ADDRESS, "#ffe187");
+        typeColorMap.put(DICTIONARY_AUTHOR, "#ffe187");
+        typeColorMap.put(DICTIONARY_SPONSOR, "#85ebff");
+        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
+        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
+        typeColorMap.put(HINT_ONLY_INDICATOR, "#abc0c4");
+        typeColorMap.put(MUST_REDACT_INDICATOR, "#fab4c0");
+        typeColorMap.put(PUBLISHED_INFORMATION_INDICATOR, "#85ebff");
+        typeColorMap.put(TEST_METHOD_INDICATOR, "#91fae8");
+        typeColorMap.put(DICTIONARY_PII, "#66ccff");
+        typeColorMap.put(PURITY_INDICATOR, "#ffe187");
+        typeColorMap.put(IMAGE_INDICATOR, "#fcc5fb");
+        typeColorMap.put(OCR_INDICATOR, "#fcc5fb");
+        typeColorMap.put(LOGO_INDICATOR, "#ffe187");
+        typeColorMap.put(FORMULA_INDICATOR, "#ffe187");
+        typeColorMap.put(SIGNATURE_INDICATOR, "#ffe187");
+        typeColorMap.put(IMPORTED_REDACTION_INDICATOR, "#fcfbe6");
+        typeColorMap.put(ROTATE_SIMPLE_INDICATOR, "#66ccff");
+
+        hintTypeMap.put(VERTEBRATE_INDICATOR, true);
+        hintTypeMap.put(DICTIONARY_ADDRESS, false);
+        hintTypeMap.put(DICTIONARY_AUTHOR, false);
+        hintTypeMap.put(DICTIONARY_SPONSOR, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY_INDICATOR, true);
+        hintTypeMap.put(MUST_REDACT_INDICATOR, true);
+        hintTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, true);
+        hintTypeMap.put(TEST_METHOD_INDICATOR, true);
+        hintTypeMap.put(DICTIONARY_PII, false);
+        hintTypeMap.put(PURITY_INDICATOR, false);
+        hintTypeMap.put(IMAGE_INDICATOR, true);
+        hintTypeMap.put(OCR_INDICATOR, true);
+        hintTypeMap.put(FORMULA_INDICATOR, false);
+        hintTypeMap.put(LOGO_INDICATOR, false);
+        hintTypeMap.put(SIGNATURE_INDICATOR, false);
+        hintTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, false);
+        hintTypeMap.put(IMPORTED_REDACTION_INDICATOR, false);
+        hintTypeMap.put(ROTATE_SIMPLE_INDICATOR, false);
+
+        caseInSensitiveMap.put(VERTEBRATE_INDICATOR, true);
+        caseInSensitiveMap.put(DICTIONARY_ADDRESS, false);
+        caseInSensitiveMap.put(DICTIONARY_AUTHOR, false);
+        caseInSensitiveMap.put(DICTIONARY_SPONSOR, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY_INDICATOR, true);
+        caseInSensitiveMap.put(MUST_REDACT_INDICATOR, true);
+        caseInSensitiveMap.put(PUBLISHED_INFORMATION_INDICATOR, true);
+        caseInSensitiveMap.put(TEST_METHOD_INDICATOR, false);
+        caseInSensitiveMap.put(DICTIONARY_PII, false);
+        caseInSensitiveMap.put(PURITY_INDICATOR, false);
+        caseInSensitiveMap.put(IMAGE_INDICATOR, true);
+        caseInSensitiveMap.put(OCR_INDICATOR, true);
+        caseInSensitiveMap.put(SIGNATURE_INDICATOR, true);
+        caseInSensitiveMap.put(LOGO_INDICATOR, true);
+        caseInSensitiveMap.put(FORMULA_INDICATOR, true);
+        caseInSensitiveMap.put(DOSSIER_REDACTIONS_INDICATOR, false);
+        caseInSensitiveMap.put(IMPORTED_REDACTION_INDICATOR, false);
+        caseInSensitiveMap.put(ROTATE_SIMPLE_INDICATOR, true);
+
+        recommendationTypeMap.put(VERTEBRATE_INDICATOR, false);
+        recommendationTypeMap.put(DICTIONARY_ADDRESS, false);
+        recommendationTypeMap.put(DICTIONARY_AUTHOR, false);
+        recommendationTypeMap.put(DICTIONARY_SPONSOR, false);
+        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(HINT_ONLY_INDICATOR, false);
+        recommendationTypeMap.put(MUST_REDACT_INDICATOR, false);
+        recommendationTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, false);
+        recommendationTypeMap.put(TEST_METHOD_INDICATOR, false);
+        recommendationTypeMap.put(DICTIONARY_PII, false);
+        recommendationTypeMap.put(PURITY_INDICATOR, false);
+        recommendationTypeMap.put(IMAGE_INDICATOR, false);
+        recommendationTypeMap.put(OCR_INDICATOR, false);
+        recommendationTypeMap.put(FORMULA_INDICATOR, false);
+        recommendationTypeMap.put(SIGNATURE_INDICATOR, false);
+        recommendationTypeMap.put(LOGO_INDICATOR, false);
+        recommendationTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, false);
+        recommendationTypeMap.put(IMPORTED_REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(ROTATE_SIMPLE_INDICATOR, false);
+
+        rankTypeMap.put(PURITY_INDICATOR, 155);
+        rankTypeMap.put(DICTIONARY_PII, 150);
+        rankTypeMap.put(DICTIONARY_ADDRESS, 140);
+        rankTypeMap.put(DICTIONARY_AUTHOR, 130);
+        rankTypeMap.put(DICTIONARY_SPONSOR, 120);
+        rankTypeMap.put(VERTEBRATE_INDICATOR, 110);
+        rankTypeMap.put(MUST_REDACT_INDICATOR, 100);
+        rankTypeMap.put(REDACTION_INDICATOR, 90);
+        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
+        rankTypeMap.put(PUBLISHED_INFORMATION_INDICATOR, 70);
+        rankTypeMap.put(TEST_METHOD_INDICATOR, 60);
+        rankTypeMap.put(HINT_ONLY_INDICATOR, 50);
+        rankTypeMap.put(IMAGE_INDICATOR, 30);
+        rankTypeMap.put(OCR_INDICATOR, 29);
+        rankTypeMap.put(LOGO_INDICATOR, 28);
+        rankTypeMap.put(SIGNATURE_INDICATOR, 27);
+        rankTypeMap.put(FORMULA_INDICATOR, 26);
+        rankTypeMap.put(DOSSIER_REDACTIONS_INDICATOR, 200);
+        rankTypeMap.put(IMPORTED_REDACTION_INDICATOR, 200);
+        rankTypeMap.put(ROTATE_SIMPLE_INDICATOR, 150);
+
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
+    }
+
+
+    @SneakyThrows
+    protected void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                responseJson.getInputStream());
+    }
+
+
+    protected List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+
+    protected Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
+                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        if (entries == null) {
+            entries = Collections.emptyList();
+        }
+        return entries.stream().map(this::toDictionaryEntry).collect(Collectors.toList());
+
+    }
+
+
+    private DictionaryEntry toDictionaryEntry(String entry) {
+
+        return DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build();
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest uploadFileToStorage(String file) {
+
+        return prepareStorage(file, "files/cv_service_empty_response.json");
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
+                cvServiceResponseFileStream);
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+
+        return request;
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -18,7 +18,9 @@ import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
@@ -27,6 +29,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -47,7 +50,7 @@ public class DictionaryServiceTest {
     protected DictionaryService dictionaryService;
 
     @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
     public static class RedactionIntegrationTestConfiguration {
 
         @Bean
@@ -63,6 +66,8 @@ public class DictionaryServiceTest {
     @Test
     public void testDictionaryServiceConsistency() {
 
+        TenantContext.setTenantId("redaction");
+
         when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
         when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId",
                 "#cccccc",

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -37,7 +37,9 @@ import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
@@ -199,7 +201,8 @@ public class HeadlinesGoldStandardIntegrationTest {
 
 
     @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class/*, StorageAutoConfiguration.class*/})
+    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
     public static class RedactionIntegrationTestConfiguration {
 
         @Bean
@@ -240,6 +243,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -3,28 +3,26 @@ package com.iqser.red.service.redaction.v1.server;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;
 
-import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
-import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.net.URI;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -34,23 +32,19 @@ import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
-import org.mockito.stubbing.Answer;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@@ -67,24 +61,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
@@ -94,85 +78,13 @@ import lombok.SneakyThrows;
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
-public class RedactionIntegrationTest {
+public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/rules.drl");
-    private static final String VERTEBRATE = "vertebrate";
-    private static final String ADDRESS = "CBI_address";
-    private static final String AUTHOR = "CBI_author";
-    private static final String SPONSOR = "CBI_sponsor";
-    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
-    private static final String REDACTION_INDICATOR = "redaction_indicator";
-    private static final String HINT_ONLY = "hint_only";
-    private static final String MUST_REDACT = "must_redact";
-    private static final String PUBLISHED_INFORMATION = "published_information";
-    private static final String TEST_METHOD = "test_method";
-    private static final String PURITY = "purity";
-    private static final String IMAGE = "image";
-    private static final String LOGO = "logo";
-    private static final String SIGNATURE = "signature";
-    private static final String FORMULA = "formula";
-    private static final String OCR = "ocr";
-    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
-    private static final String IMPORTED_REDACTION = "imported_redaction";
-    private static final String PII = "PII";
-    private static final String ROTATE_SIMPLE = "RotateSimple";
-
-    @Autowired
-    private RedactionController redactionController;
-
-    @Autowired
-    private AnnotationService annotationService;
-
-    @Autowired
-    private AnalyzeService analyzeService;
-
-    @Autowired
-    private ObjectMapper objectMapper;
-
-    @MockBean
-    private RulesClient rulesClient;
-
-    @MockBean
-    private DictionaryClient dictionaryClient;
-
-    @Autowired
-    private RedactionStorageService redactionStorageService;
-
-    @Autowired
-    private StorageService storageService;
-
-    @Autowired
-    private ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
-
-    @MockBean
-    private AmazonS3 amazonS3;
-
-    @MockBean
-    private RabbitTemplate rabbitTemplate;
-
-    @MockBean
-    private LegalBasisClient legalBasisClient;
-
-    private final Map<String, List<String>> dictionary = new HashMap<>();
-    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
-    private final Map<String, List<String>> falsePositive = new HashMap<>();
-    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
-    private final Map<String, String> typeColorMap = new HashMap<>();
-    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
-    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
-    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
-    private final Map<String, Integer> rankTypeMap = new HashMap<>();
-    private final Colors colors = new Colors();
-    private final Map<String, Long> reanlysisVersions = new HashMap<>();
-    private final Set<String> deleted = new HashSet<>();
-
-    private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
-    private final static String TEST_DOSSIER_ID = "123";
-    private final static String TEST_FILE_ID = "123";
 
     @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
     public static class RedactionIntegrationTestConfiguration {
 
         @Bean
@@ -201,18 +113,11 @@ public class RedactionIntegrationTest {
     }
 
 
-    @AfterEach
-    public void cleanupStorage() {
-
-        if (this.storageService instanceof FileSystemBackedStorageService) {
-            ((FileSystemBackedStorageService) this.storageService).clearStorage();
-        }
-    }
-
-
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
+
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
 
@@ -224,67 +129,26 @@ public class RedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS)
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
                 .dossierTemplateId(TEST_DOSSIER_ID)
                 .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
                 .build()));
 
         mockDictionaryCalls(null);
-        mockDictionaryCalls(0L);
 
         when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
     }
 
 
-    private void mockDictionaryCalls(Long version) {
-
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
-                false));
-        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                NO_REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                REDACTION_INDICATOR,
-                false));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
-                false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                PUBLISHED_INFORMATION,
-                false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
-                false));
-        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
-        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
-        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
-        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
-        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
-        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
-                false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                DOSSIER_REDACTIONS,
-                true));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-                IMPORTED_REDACTION,
-                true));
-
-    }
-
-
     @Test
     public void test270Rotated() {
 
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/270Rotated.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
         assertThat(result).isNotNull();
@@ -295,7 +159,7 @@ public class RedactionIntegrationTest {
     @Disabled
     public void testLargeScannedFileOOM() {
 
-        AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
+        AnalyzeRequest request = uploadFileToStorage("scanned/VV-377031.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
         assertThat(result).isNotNull();
@@ -307,7 +171,7 @@ public class RedactionIntegrationTest {
 
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/merge_images.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/merge_images.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
@@ -321,7 +185,7 @@ public class RedactionIntegrationTest {
 
         duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));
 
-        dictionary.get(AUTHOR).add("Drinking water");
+        dictionary.get(DICTIONARY_AUTHOR).add("Drinking water");
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
@@ -347,7 +211,7 @@ public class RedactionIntegrationTest {
 
         // F. Lastname, J. Doe, M. Mustermann
         // Lastname M., Doe J., Mustermann M.
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/ExpansionTest.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/ExpansionTest.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -363,7 +227,7 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -386,7 +250,7 @@ public class RedactionIntegrationTest {
     @Test
     public void testAddFileAttribute() {
 
-        AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -403,7 +267,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("testIgnoreHint");
 
-        AnalyzeRequest request = prepareStorage("files/new/test-ignore-hint.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/test-ignore-hint.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         analyzeService.analyze(request);
@@ -441,15 +305,11 @@ public class RedactionIntegrationTest {
         System.out.println("noExceptionShouldBeThrownForAnyFiles");
         ClassLoader loader = getClass().getClassLoader();
         URL url = loader.getResource("files");
-        File[] files = new File(url.getPath()).listFiles();
-        List<File> input = new ArrayList<>();
-        for (File file : files) {
-            input.addAll(getPathsRecursively(file));
-        }
-        for (File path : input) {
+        Path path = Paths.get(URI.create(url.toString()));
 
-            AnalyzeRequest request = prepareStorage(path.getPath());
-            System.out.println("Redacting file : " + path.getName());
+        Files.walk(path).forEach(currentPath -> {
+            AnalyzeRequest request = uploadFileToStorage(currentPath.toString());
+            System.out.println("Redacting file : " + currentPath.getFileName());
             analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
 
             long fstart = System.currentTimeMillis();
@@ -468,7 +328,7 @@ public class RedactionIntegrationTest {
                 assertThat(entry.getValue().size()).isEqualTo(1);
             });
 
-            dictionary.get(AUTHOR).add("Drinking water");
+            dictionary.get(DICTIONARY_AUTHOR).add("Drinking water");
             when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
 
             long rstart = System.currentTimeMillis();
@@ -476,8 +336,7 @@ public class RedactionIntegrationTest {
 
             long rend = System.currentTimeMillis();
             System.out.println("reanalysis analysis duration: " + (rend - rstart));
-
-        }
+        });
 
         long end = System.currentTimeMillis();
 
@@ -493,7 +352,7 @@ public class RedactionIntegrationTest {
         String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf";
 
         long start = System.currentTimeMillis();
-        AnalyzeRequest request = prepareStorage(fileName);
+        AnalyzeRequest request = uploadFileToStorage(fileName);
         request.setExcludedPages(Set.of(1));
 
         request.setFileAttributes(List.of(FileAttribute.builder()
@@ -537,10 +396,10 @@ public class RedactionIntegrationTest {
         }
         assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size());
 
-        dictionary.get(AUTHOR).add("properties");
+        dictionary.get(DICTIONARY_AUTHOR).add("properties");
         reanlysisVersions.put("properties", 1L);
 
-        dictionary.get(AUTHOR).add("physical");
+        dictionary.get(DICTIONARY_AUTHOR).add("physical");
         reanlysisVersions.put("physical", 2L);
 
         deleted.add("David Chubb");
@@ -550,7 +409,7 @@ public class RedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
 
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
 
         start = System.currentTimeMillis();
 
@@ -583,7 +442,7 @@ public class RedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
 
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
 
         analyzeService.reanalyze(request);
 
@@ -599,19 +458,19 @@ public class RedactionIntegrationTest {
         String fileName = "files/new/test1S1T1.pdf";
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
-        AnalyzeRequest request = prepareStorage(fileName);
+        AnalyzeRequest request = uploadFileToStorage(fileName);
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         analyzeService.analyze(request);
 
-        dictionary.get(AUTHOR).add("report");
+        dictionary.get(DICTIONARY_AUTHOR).add("report");
         reanlysisVersions.put("report", 2L);
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
         mockDictionaryCalls(0L);
 
         analyzeService.reanalyze(request);
 
-        dictionary.get(AUTHOR).add("assessment report");
+        dictionary.get(DICTIONARY_AUTHOR).add("assessment report");
         reanlysisVersions.put("assessment report", 3L);
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
         mockDictionaryCalls(2L);
@@ -642,10 +501,12 @@ public class RedactionIntegrationTest {
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
         ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                responseJson.getInputStream());
 
         long start = System.currentTimeMillis();
-        AnalyzeRequest request = prepareStorage(fileName);
+        AnalyzeRequest request = uploadFileToStorage(fileName);
         request.setExcludedPages(Set.of(1));
 
         request.setFileAttributes(List.of(FileAttribute.builder()
@@ -689,11 +550,11 @@ public class RedactionIntegrationTest {
         }
         assertThat(correctFound).isEqualTo(redactionLog.getRedactionLogEntry().size());
 
-        dictionary.get(AUTHOR).add("properties");
+        dictionary.get(DICTIONARY_AUTHOR).add("properties");
         reanlysisVersions.put("properties", 1L);
         mockDictionaryCalls(0L);
 
-        dictionary.get(AUTHOR).add("physical");
+        dictionary.get(DICTIONARY_AUTHOR).add("physical");
         reanlysisVersions.put("physical", 2L);
 
         deleted.add("David Chubb");
@@ -705,7 +566,7 @@ public class RedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
 
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
 
         start = System.currentTimeMillis();
 
@@ -739,7 +600,7 @@ public class RedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
 
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE_INDICATOR, null)).thenReturn(getDictionaryResponse(VERTEBRATE_INDICATOR, false));
 
         analyzeService.reanalyze(request);
 
@@ -762,7 +623,7 @@ public class RedactionIntegrationTest {
         };
         List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
 
-        AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/PublishedInformationTest.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         ManualRedactions manualRedactions = new ManualRedactions();
         manualRedactions.getIdsToRemove()
@@ -820,7 +681,7 @@ public class RedactionIntegrationTest {
         System.out.println("testTableRedaction");
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
@@ -889,7 +750,7 @@ public class RedactionIntegrationTest {
 
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/new/unicodeProblem.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/unicodeProblem.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
@@ -911,7 +772,7 @@ public class RedactionIntegrationTest {
         System.out.println("testTableRedaction");
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/RotateTestFile.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
@@ -937,7 +798,7 @@ public class RedactionIntegrationTest {
         System.out.println("testTableRedaction");
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFileSimple.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/RotateTestFileSimple.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
@@ -959,7 +820,7 @@ public class RedactionIntegrationTest {
         System.out.println("testTableHeader");
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/NoHeaderTable.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/NoHeaderTable.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
@@ -994,7 +855,7 @@ public class RedactionIntegrationTest {
 
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/new/S157.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/S157.pdf");
 
         ManualRedactions manualRedactions = new ManualRedactions();
 
@@ -1070,7 +931,7 @@ public class RedactionIntegrationTest {
         manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
                 Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
 
-        AnalyzeRequest request = prepareStorage(pdfFile);
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
         request.setManualRedactions(manualRedactions);
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1100,14 +961,10 @@ public class RedactionIntegrationTest {
     }
 
 
-
-
-
-
     @Test
     public void phantomCellsDocumentTest() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/Phantom Cells.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/Phantom Cells.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1127,7 +984,7 @@ public class RedactionIntegrationTest {
 
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/sponsor_companies.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/sponsor_companies.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1189,7 +1046,7 @@ public class RedactionIntegrationTest {
 
 //        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
 
-        AnalyzeRequest request = prepareStorage(pdfFile);
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
         request.setManualRedactions(manualRedactions);
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1245,7 +1102,7 @@ public class RedactionIntegrationTest {
         System.out.println("expandByRegex");
         long start = System.currentTimeMillis();
 
-        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
@@ -1300,7 +1157,7 @@ public class RedactionIntegrationTest {
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
 
-        AnalyzeRequest request = prepareStorage(pdfFile);
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
         request.setManualRedactions(manualRedactions);
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1324,280 +1181,15 @@ public class RedactionIntegrationTest {
     }
 
 
-    private void loadDictionaryForTest() {
-
-        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
-
-        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-
-    }
-
-
-    private void loadOnlyDictionaryForSimpleFile() {
-
-        dictionary.clear();
-        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-    }
-
-
-    private static String loadFromClassPath(String path) {
-
-        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
-        if (resource == null) {
-            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
-        }
-        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
-            StringBuilder sb = new StringBuilder();
-            String str;
-            while ((str = br.readLine()) != null) {
-                sb.append(str).append("\n");
-            }
-            return sb.toString();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
-        }
-    }
-
-
-    private List<File> getPathsRecursively(File path) {
-
-        List<File> result = new ArrayList<>();
-        if (path == null || path.listFiles() == null) {
-            return result;
-        }
-        for (File f : path.listFiles()) {
-            if (f.isFile()) {
-                result.add(f);
-            } else {
-                result.addAll(getPathsRecursively(f));
-            }
-        }
-        return result;
-
-    }
-
-
-    private void loadTypeForTest() {
-
-        typeColorMap.put(VERTEBRATE, "#ff85f7");
-        typeColorMap.put(ADDRESS, "#ffe187");
-        typeColorMap.put(AUTHOR, "#ffe187");
-        typeColorMap.put(SPONSOR, "#85ebff");
-        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
-        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
-        typeColorMap.put(HINT_ONLY, "#abc0c4");
-        typeColorMap.put(MUST_REDACT, "#fab4c0");
-        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
-        typeColorMap.put(TEST_METHOD, "#91fae8");
-        typeColorMap.put(PII, "#66ccff");
-        typeColorMap.put(PURITY, "#ffe187");
-        typeColorMap.put(IMAGE, "#fcc5fb");
-        typeColorMap.put(OCR, "#fcc5fb");
-        typeColorMap.put(LOGO, "#ffe187");
-        typeColorMap.put(FORMULA, "#ffe187");
-        typeColorMap.put(SIGNATURE, "#ffe187");
-        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
-        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
-
-        hintTypeMap.put(VERTEBRATE, true);
-        hintTypeMap.put(ADDRESS, false);
-        hintTypeMap.put(AUTHOR, false);
-        hintTypeMap.put(SPONSOR, false);
-        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
-        hintTypeMap.put(REDACTION_INDICATOR, true);
-        hintTypeMap.put(HINT_ONLY, true);
-        hintTypeMap.put(MUST_REDACT, true);
-        hintTypeMap.put(PUBLISHED_INFORMATION, true);
-        hintTypeMap.put(TEST_METHOD, true);
-        hintTypeMap.put(PII, false);
-        hintTypeMap.put(PURITY, false);
-        hintTypeMap.put(IMAGE, true);
-        hintTypeMap.put(OCR, true);
-        hintTypeMap.put(FORMULA, false);
-        hintTypeMap.put(LOGO, false);
-        hintTypeMap.put(SIGNATURE, false);
-        hintTypeMap.put(DOSSIER_REDACTIONS, false);
-        hintTypeMap.put(IMPORTED_REDACTION, false);
-        hintTypeMap.put(ROTATE_SIMPLE, false);
-
-        caseInSensitiveMap.put(VERTEBRATE, true);
-        caseInSensitiveMap.put(ADDRESS, false);
-        caseInSensitiveMap.put(AUTHOR, false);
-        caseInSensitiveMap.put(SPONSOR, false);
-        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(HINT_ONLY, true);
-        caseInSensitiveMap.put(MUST_REDACT, true);
-        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
-        caseInSensitiveMap.put(TEST_METHOD, false);
-        caseInSensitiveMap.put(PII, false);
-        caseInSensitiveMap.put(PURITY, false);
-        caseInSensitiveMap.put(IMAGE, true);
-        caseInSensitiveMap.put(OCR, true);
-        caseInSensitiveMap.put(SIGNATURE, true);
-        caseInSensitiveMap.put(LOGO, true);
-        caseInSensitiveMap.put(FORMULA, true);
-        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
-        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
-        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
-
-        recommendationTypeMap.put(VERTEBRATE, false);
-        recommendationTypeMap.put(ADDRESS, false);
-        recommendationTypeMap.put(AUTHOR, false);
-        recommendationTypeMap.put(SPONSOR, false);
-        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(REDACTION_INDICATOR, false);
-        recommendationTypeMap.put(HINT_ONLY, false);
-        recommendationTypeMap.put(MUST_REDACT, false);
-        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
-        recommendationTypeMap.put(TEST_METHOD, false);
-        recommendationTypeMap.put(PII, false);
-        recommendationTypeMap.put(PURITY, false);
-        recommendationTypeMap.put(IMAGE, false);
-        recommendationTypeMap.put(OCR, false);
-        recommendationTypeMap.put(FORMULA, false);
-        recommendationTypeMap.put(SIGNATURE, false);
-        recommendationTypeMap.put(LOGO, false);
-        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
-        recommendationTypeMap.put(IMPORTED_REDACTION, false);
-        recommendationTypeMap.put(ROTATE_SIMPLE, false);
-
-        rankTypeMap.put(PURITY, 155);
-        rankTypeMap.put(PII, 150);
-        rankTypeMap.put(ADDRESS, 140);
-        rankTypeMap.put(AUTHOR, 130);
-        rankTypeMap.put(SPONSOR, 120);
-        rankTypeMap.put(VERTEBRATE, 110);
-        rankTypeMap.put(MUST_REDACT, 100);
-        rankTypeMap.put(REDACTION_INDICATOR, 90);
-        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
-        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
-        rankTypeMap.put(TEST_METHOD, 60);
-        rankTypeMap.put(HINT_ONLY, 50);
-        rankTypeMap.put(IMAGE, 30);
-        rankTypeMap.put(OCR, 29);
-        rankTypeMap.put(LOGO, 28);
-        rankTypeMap.put(SIGNATURE, 27);
-        rankTypeMap.put(FORMULA, 26);
-        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
-        rankTypeMap.put(IMPORTED_REDACTION, 200);
-        rankTypeMap.put(ROTATE_SIMPLE, 150);
-
-        colors.setSkippedColor("#cccccc");
-        colors.setRequestAddColor("#04b093");
-        colors.setRequestRemoveColor("#04b093");
-    }
-
-
-    @SneakyThrows
-    private void loadNerForTest() {
-
-        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
-    }
-
-
-    private List<Type> getTypeResponse() {
-
-        return typeColorMap.entrySet()
-                .stream()
-                .map(typeColor -> Type.builder()
-                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                        .type(typeColor.getKey())
-                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                        .hexColor(typeColor.getValue())
-                        .isHint(hintTypeMap.get(typeColor.getKey()))
-                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
-                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
-                        .rank(rankTypeMap.get(typeColor.getKey()))
-                        .build())
-
-                .collect(Collectors.toList());
-    }
-
-
-    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
-
-        return Type.builder()
-                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .hexColor(typeColorMap.get(type))
-                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
-                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
-                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
-                .isHint(hintTypeMap.get(type))
-                .isCaseInsensitive(caseInSensitiveMap.get(type))
-                .isRecommendation(recommendationTypeMap.get(type))
-                .rank(rankTypeMap.get(type))
-                .build();
-    }
-
-
-    private String cleanDictionaryEntry(String entry) {
-
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-
-
-    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
-
-        if (entries == null) {
-            entries = Collections.emptyList();
-        }
-
-        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
-        entries.forEach(entry -> {
-            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
-        });
-        return dictionaryEntries;
-    }
-
-
     @Test
     public void testImportedRedactions() throws IOException {
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
         ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
 
-        AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+        AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
                 importedRedactions.getInputStream());
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
@@ -1628,17 +1220,17 @@ public class RedactionIntegrationTest {
     @Test
     public void testExpandByPrefixRegEx() throws IOException {
 
-        assertThat(dictionary.get(AUTHOR)).contains("Robinson");
-        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mrs. Robinson");
-        assertThat(dictionary.get(AUTHOR)).contains("Bojangles");
-        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Bojangles");
-        assertThat(dictionary.get(AUTHOR)).contains("Tambourine Man");
-        assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Tambourine Man");
+        assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Robinson");
+        assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mrs. Robinson");
+        assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Bojangles");
+        assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mr. Bojangles");
+        assertThat(dictionary.get(DICTIONARY_AUTHOR)).contains("Tambourine Man");
+        assertThat(dictionary.get(DICTIONARY_AUTHOR)).doesNotContain("Mr. Tambourine Man");
 
         String fileName = "files/mr-mrs.pdf";
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
-        AnalyzeRequest request = prepareStorage(fileName);
+        AnalyzeRequest request = uploadFileToStorage(fileName);
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         analyzeService.analyze(request);
@@ -1657,39 +1249,4 @@ public class RedactionIntegrationTest {
         assertThat(values).contains("Mr. Tambourine Man");
     }
 
-
-    @SneakyThrows
-    private AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
-
-        AnalyzeRequest request = AnalyzeRequest.builder()
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .lastProcessed(OffsetDateTime.now())
-                .build();
-
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
-
-        return request;
-
-    }
-
-
-    @SneakyThrows
-    private AnalyzeRequest prepareStorage(String file) {
-
-        return prepareStorage(file, "files/cv_service_empty_response.json");
-    }
-
-
-    @SneakyThrows
-    private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
-
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
-        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
-
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java

@@ -0,0 +1,176 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class)
+
+public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest {
+
+    private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("redaction");
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    /**
+     * The case in this test: One term, 'Dr. Alan Miller', is found by PII-Rule and is in the PII-dictionary
+     * as well as in the PII-false-positive-list - and in the CBI-author dictionary.
+     * It gets redacted, as the PII-finding is false positive and so the CBI-author entry is effective,
+     * independent of the entity-rank
+     */
+    @Test
+    @SneakyThrows
+    public void testTermIsInTwoDictionariesAndInOneFalsePositive() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/new/simplified2.pdf");
+
+        String entryAuthorAndPIIDictionary = "Dr. Alan Miller";
+
+        dictionary.get(DICTIONARY_AUTHOR).add(entryAuthorAndPIIDictionary);
+        dictionary.put(DICTIONARY_PII, List.of(entryAuthorAndPIIDictionary));
+
+        String entryPIIFalsePositive = "Dr. Alan Miller COMPLETION DATE:";
+
+        falsePositive.put(DICTIONARY_PII, List.of(entryPIIFalsePositive));
+
+        reanlysisVersions.put(entryAuthorAndPIIDictionary, 1L);
+        reanlysisVersions.put(entryPIIFalsePositive, 1L);
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
+        mockDictionaryCalls(0L);
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(1);
+
+        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
+
+        assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
+        assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorAndPIIDictionary);
+        assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
+        assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
+        assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
+        assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
+        assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
+
+        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
+        assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
+
+    }
+
+
+    /**
+     * The case in this test: The term 'Evans P.G.' is very close to a table-cell. It will get redacted nevertheless.
+     */
+    @Test
+    @SneakyThrows
+    public void testTermGetsRedactedEvenItsCloseToCellBorder() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/SinglePages/VV-931175_Page1.pdf");
+
+        dictionary.clear();
+        falsePositive.clear();
+
+        String entryAuthorDictionary = "Evans P.G.";
+        dictionary.put(DICTIONARY_AUTHOR, List.of(entryAuthorDictionary));
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(1);
+
+        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
+
+        assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
+        assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorDictionary);
+        assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
+        assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
+        assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
+        assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
+        assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
+
+        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
+        assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -46,7 +46,9 @@ import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
@@ -245,6 +247,7 @@ public class RulesTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
         objectMapper.registerModule(new JavaTimeModule());
         objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
@@ -801,7 +804,8 @@ public class RulesTest {
 
 
     @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
     public static class RulesTestConfiguration {
 
         @Bean

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequenceTest.java

@@ -49,18 +49,6 @@ public class TextPositionSequenceTest {
         assertThat(textPositionSequence.getPageWidth()).isEqualTo(600f);
     }
 
-
-    @Test
-    @SneakyThrows
-    public void testDeserializationWithDslJson() {
-
-        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithDslJson(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
-                TextPositionSequence.class);
-
-        assertPropertiesAfterJsonDeserialization(textPositionSequence);
-    }
-
-
     private Matrix createIdentityMatrix() {
 
         return new Matrix();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -19,10 +19,13 @@ import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
@@ -42,6 +45,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@@ -55,6 +59,7 @@ import lombok.SneakyThrows;
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
+@AutoConfigureObservability
 public class LiveDataIntegrationTest {
 
     protected static String BASE_DIR = "performance/";
@@ -97,7 +102,8 @@ public class LiveDataIntegrationTest {
     private List<Type> types;
 
     @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters={@ComponentScan.Filter(type= FilterType.ASSIGNABLE_TYPE, value=StorageAutoConfiguration.class)})
     public static class RedactionIntegrationTestConfiguration {
 
         @Bean
@@ -114,6 +120,7 @@ public class LiveDataIntegrationTest {
     @BeforeEach
     public void prepareTest() {
 
+        TenantContext.setTenantId("redaction");
         when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
         when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -21,7 +22,9 @@ import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
@@ -89,7 +92,8 @@ public class PdfSegmentationServiceTest {
     private final static String TEST_FILE_ID = "123";
 
     @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
     public static class TestConfiguration {
 
         @Bean
@@ -102,6 +106,14 @@ public class PdfSegmentationServiceTest {
     }
 
 
+    @SneakyThrows
+    private void prepareStorage() {
+
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
+                new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
+    }
+
     @Test
     @SneakyThrows
     public void testMapping() {
@@ -133,7 +145,7 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
         assertThat(table.getColCount()).isEqualTo(6);
         assertThat(table.getRowCount()).isEqualTo(13);
         assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@@ -148,14 +160,14 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
-        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).collect(Collectors.toList()).equals(firstTableHeaderCells))).isTrue();
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
@@ -167,14 +179,14 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
-        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).collect(Collectors.toList()).equals(firstTableHeaderCells))).isTrue();
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
@@ -186,22 +198,406 @@ public class PdfSegmentationServiceTest {
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
         assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
-        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).collect(Collectors.toList()).equals(firstTableHeaderCells))).isTrue();
+        assertThat(secondTable.getRows().stream().allMatch(row -> row.stream().map(Cell::getHeaderCells).toList().equals(firstTableHeaderCells))).isTrue();
     }
 
 
-    @SneakyThrows
-    private void prepareStorage() {
+    @Test
+    public void testDoc56Page170() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 1, 1, 0, 0);
+        validateTable(document, 1, 2, 2, 0, 0);
+        validateTable(document, 2, 7, 20, 0, 140);
+        validateTable(document, 3, 8, 31, 0, 170);
 
-        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES),
-                new ClassPathResource("files/cv_service_empty_response.json").getInputStream());
     }
 
+
+    @Test
+    public void testVV931175Page1() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/VV-931175_Page1.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 8, 9, 0, 2);
+
+        List<List<String>> values = Arrays.asList(
+                Arrays.asList(
+                "Annex point Reference within DAR/RAR",
+                        "Author, date",
+                        "Study title",
+                        "Analytical method Author, date, No.",
+                        "Technique, LOQ of the method, validated working range",
+                        "Method meets analytical validation criteria",
+                        "Remarks (in case validation criteria are not met)",
+                        "Acceptability of the method"
+                ),
+                Arrays.asList(
+                "",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"
+                ),
+                Arrays.asList(
+                        "CA 7.1.2.1.1 DAR (2009)",
+                        "Evans P.G. 2001 TMJ4569B, VV-323245",
+                        "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
+                        "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
+                        "Y",
+                        "N/A",
+                        "Y"
+                )
+        );
+
+        validateTable(document, 0, values);
+
+    }
+
+
+    @Test
+    public void testDoc27Page6() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/27 A8637C - EU AIR3 - MCP Section 1 - Identity of the plant protection product_Page6.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 3, 2, 0, 0);
+        validateTable(document, 1, 3, 2, 0, 0);
+        validateTable(document, 2, 3, 3, 0, 0);
+        validateTable(document, 3, 3, 3, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDocA20622APartB9Page185() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 5, 5, 0, 23);
+        validateTable(document, 1, 11, 9, 0, 36);
+
+    }
+
+
+    @Test
+    public void testDocA20622APartB7Page123() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izZRMS (CZ) fRR Part B7_Page123.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 6);
+
+        validateTable(document, 0, 2, 1, 0, 0);
+        validateTable(document, 1, 2, 1, 0, 0);
+        validateTable(document, 2, 2, 5, 0, 0);
+        validateTable(document, 3, 2, 5, 0, 0);
+        validateTable(document, 4, 2, 4, 0, 0);
+        validateTable(document, 5, 2, 1, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc77Page111() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/77 Pirimicarb_RAR_08_Volume_3CA_B-6_2017-12-04_Page11.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 3);
+
+        validateTable(document, 0, 7, 9, 0, 0);
+        validateTable(document, 1, 2, 1, 0, 0);
+        validateTable(document, 2, 2, 10, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc95Page532() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10_Page532.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+        validateTable(document, 0, 9, 9, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc52Page175() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page175.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 9, 5, 6, 0);
+
+    }
+
+
+    @Test
+    public void testDoc52Page174() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21_Page174.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+        validateTable(document, 0, 9, 6, 7, 0);
+
+    }
+
+
+    @Test
+    public void testDoc19Page35() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page35.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+        validateTable(document, 0, 10, 6, 0, 1);
+
+    }
+
+
+    @Test
+    public void testDoc19Page161() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/19 Chlorothalonil RAR 08 Volume 3CA B 6b metabolites Oct 2017_Page161.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 2);
+        validateTable(document, 0, 2, 2, 0, 0);
+        validateTable(document, 1, 1, 1, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc47Page30() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource(
+                "files/SinglePages/47 Cyprodinil - EU AIR3 - MCA Section 5 Supplement - Toxicological and metabolism studies on the active substance_Page30.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 7, 8, 1, 0);
+        validateTable(document, 1, 7, 8, 1, 0);
+
+    }
+
+
+    @Test
+    public void testDoc49Page61() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource(
+                "files/SinglePages/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance_Page61.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 4, 17, 0, 0);
+        validateTable(document, 1, 7, 12, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc81Page54() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/81 Pirimicarb_RAR_20_Volume_3CP_A10788A (_Pirimor_)_B-9_2017-12-04_Page54.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 5, 14, 4, 0);
+        validateTable(document, 1, 7, 12, 0, 0);
+
+    }
+
+
+    @Test
+    public void testDoc88Page134() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26_Page134.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 5, 17, 3, 0);
+        validateTable(document, 1, 5, 16, 2, 0);
+
+    }
+
+
+    @Test
+    public void testDocThiabendazolePage18() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/Thiabendazole DAR Addendum for ED_April_2020_Page18.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 4);
+
+        validateTable(document, 0, 4, 4, 0, 0);
+        validateTable(document, 1, 1, 1, 0, 0);
+        validateTable(document, 2, 2, 3, 0, 0);
+        validateTable(document, 3, 1, 1, 0, 0);
+
+    }
+
+    @Test
+    public void testDoc15Page18() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/15 - Pretilachlor - Acute Oral Toxicity (Up and Down Procedure) - Rat_Page18.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 11, 8, 0, 0);
+
+    }
+
+    @Test
+    public void testDoc28Page23() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/28 A8637C - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product_Page23.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 2);
+
+        validateTable(document, 0, 6, 8, 0, 2);
+        validateTable(document, 1, 6, 8, 0, 1);
+
+
+    }
+
+
+    @Test
+    public void testDoc24Page17() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/24 - SYN549522 - Acute Oral Toxicity - Rats_Page17.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 9, 5, 2, 0);
+
+    }
+
+    @Test
+    public void testDoc30Page5() throws IOException {
+
+        prepareStorage();
+        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/30 - Dicamba - Acute Oral Toxicity - Rats_Page5.pdf");
+
+        Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
+
+        validateTableSize(document, 1);
+
+        validateTable(document, 0, 3, 5, 0, 0);
+
+    }
+
+
+
+    private void validateTable(Document document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
+
+        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        List<List<Cell>> rows = table.getRows();
+        int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
+
+        assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);
+
+        assertThat(table.getColCount()).isEqualTo(colCount);
+        assertThat(table.getRowCount()).isEqualTo(rowCount);
+
+    }
+    private void validateTable(Document document, int tableIndex, List<List<String>> values) {
+
+        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
+        List<List<Cell>> rows = table.getRows();
+
+        List<Cell> rowsFlattened = rows.stream().flatMap(List::stream).toList();
+        List<String> valuesFlattened = values.stream().flatMap(List::stream).toList();
+
+        for (int i = 0; i < valuesFlattened.size(); i++) {
+            Cell cell = rowsFlattened.get(i);
+            String value = valuesFlattened.get(i);
+            assertThat(cell.toString()).isEqualTo(value);
+        }
+
+    }
+
+    private void validateTableSize(Document document, int tableSize) {
+
+        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().size()).isEqualTo(tableSize);
+
+    }
+
+
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -0,0 +1,341 @@
+package drools
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+
+global Section section
+
+
+// --------------------------------------- AI rules -------------------------------------------------------------------
+
+rule "0: Add CBI_author from ai"
+    when
+        Section(aiMatchesType("CBI_author"))
+    then
+        section.addAiEntities("CBI_author", "CBI_author");
+    end
+
+rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
+    when
+        Section(aiMatchesType("ORG"))
+    then
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
+    when
+        Section(aiMatchesType("STREET"))
+    then
+        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
+    when
+        Section(aiMatchesType("CITY"))
+    then
+        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
+    end
+
+/* Syngenta specific laboratory recommendation */
+rule "0: Recommend CTL/BL laboratory that start with BL or CTL"
+    when
+        Section(searchText.contains("CT") || searchText.contains("BL"))
+    then
+        /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
+        section.addRecommendationByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address");
+    end
+
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
+
+rule "1: Redact CBI Authors (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "2: Redact CBI Authors (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "3: Redact not CBI Address (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        section.ignoreRecommendations("CBI_address");
+    end
+
+rule "4: Redact CBI Address (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "5: Do not redact genitive CBI_author"
+    when
+        Section(matchesType("CBI_author"))
+    then
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+    end
+
+
+rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "16: Add recommendation for Addresses in Test Organism sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
+    then
+        section.recommendLineAfter("Source:", "CBI_address");
+    end
+
+rule "17: Add recommendation for Addresses in Test Animals sections"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
+    then
+        section.recommendLineAfter("Source", "CBI_address");
+    end
+
+
+rule "18: Do not redact Names and Addresses if Published Information found"
+    when
+        Section(matchesType("published_information"))
+    then
+        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
+        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
+    end
+
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+
+rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "21: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "22: Redact Emails by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+               text.contains("Contact")
+            || text.contains("Telephone")
+            || text.contains("Phone")
+            || text.contains("Fax")
+            || text.contains("Tel")
+            || text.contains("Ter")
+            || text.contains("Mobile")
+            || text.contains("Fel")
+            || text.contains("Fer")
+        ))
+    then
+        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+               text.contains("Contact")
+            || text.contains("Telephone")
+            || text.contains("Phone")
+            || text.contains("Fax")
+            || text.contains("Tel")
+            || text.contains("Ter")
+            || text.contains("Mobile")
+            || text.contains("Fel")
+            || text.contains("Fer")
+        ))
+    then
+        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "27: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("COMPLETION DATE:")
+            && !searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "28: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("COMPLETION DATE:")
+            && !searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "29: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "30: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("AUTHOR(S):")
+            && searchText.contains("STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("PERFORMING LABORATORY:")
+        )
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
+    end
+
+rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText.contains("PERFORMING LABORATORY:"))
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "33: Purity Hint"
+    when
+        Section(searchText.toLowerCase().contains("purity"))
+    then
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+    end
+
+
+rule "34: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction"));
+    then
+        section.ignore("dossier_redaction");
+    end
+
+
+rule "35: Redact signatures (Non vertebrate study)"
+    when
+        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "36: Redact signatures (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "43: Redact Logos (Vertebrate study)"
+    when
+        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
+    then
+        section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end