RED-4799

Merge in RED/redaction-service from RED-4799-33x-release-fix to release/3.107.x

* commit '4289dfee2ddc1acf673ce7d16569ab514d1bbe6e':
  RED-4799

bamboo-specs/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.atlassian.bamboo</groupId>
     <artifactId>bamboo-specs-parent</artifactId>
-    <version>8.0.3</version>
+    <version>8.1.3</version>
     <relativePath/>
   </parent>
 

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -17,6 +17,7 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
+import com.atlassian.bamboo.specs.api.builders.Variable;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
@@ -83,6 +84,7 @@ public class PlanSpec {
                 project(),
                 SERVICE_NAME, new BambooKey(SERVICE_KEY))
                 .description("Plan created from (enter repository url of your plan)")
+                .variables(new Variable("maven_add_param", ""))
                 .stages(new Stage("Default Stage")
                         .jobs(new Job("Default Job",
                                 new BambooKey("JOB1"))
@@ -94,6 +96,7 @@ public class PlanSpec {
                                                         "rm -rf ./*"),
                                         new VcsCheckoutTask()
                                                 .description("Checkout Default Repository")
+                                                .cleanCheckout(true)
                                                 .checkoutItems(new CheckoutItem().defaultRepository()),
                                         new ScriptTask()
                                                 .description("Build")
@@ -115,7 +118,7 @@ public class PlanSpec {
                                                 .defaultRepository())
                                 .dockerConfiguration(
                                         new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                                 .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                                 .volume("/var/run/docker.sock", "/var/run/docker.sock")
                                 )
@@ -147,6 +150,7 @@ public class PlanSpec {
                                                         "rm -rf ./*"),
                                         new VcsCheckoutTask()
                                                 .description("Checkout Default Repository")
+                                                .cleanCheckout(true)
                                                 .checkoutItems(new CheckoutItem().defaultRepository()),
                                         new ScriptTask()
                                                 .description("Sonar")
@@ -155,9 +159,9 @@ public class PlanSpec {
                                                 .argument(SERVICE_NAME))
                                 .dockerConfiguration(
                                         new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
                                                 .dockerRunArguments("--net=host")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
                                                 .volume("/var/run/docker.sock", "/var/run/docker.sock")
                                 )
                         )
@@ -165,8 +169,7 @@ public class PlanSpec {
                 .linkedRepositories("RED / " + SERVICE_NAME)
                 .triggers(
                         new ScheduledTrigger()
-                                .scheduleOnceDaily(LocalTime.of(12, 00)),
-                        new BitbucketServerTrigger())
+                                .scheduleOnceDaily(LocalTime.of(23, 00)))
                 .planBranchManagement(new PlanBranchManagement()
                         .createForVcsBranchMatching("release.*")
                         .notificationForCommitters());

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -3,49 +3,62 @@ set -e
 
 SERVICE_NAME=$1
 
-if [[ "${bamboo_version_tag}" = "dev" ]]
-then 
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+if [[ "$bamboo_planRepository_branchName" == "master" ]]
+then
+    branchVersion=$(cat pom.xml | grep -Eo "    <version>.*-SNAPSHOT</version>" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|' | tr -d ' ')
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
+    newVersion="$(semver $latestVersion -p -i minor)"
+    echo "new release on master with version $newVersion"
+elif [[ "$bamboo_planRepository_branchName" == release* ]]
+then
+    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
+    newVersion="$(semver $latestVersion -p -i patch)"
+    echo "new release on $bamboo_planRepository_branchName with version $newVersion"
+elif [[ "${bamboo_version_tag}" != "dev" ]]
+then
+    newVersion="${bamboo_version_tag}"
+    echo "new special version bild with $newVersion"
+else
+    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
         --no-transfer-progress \
+        ${bamboo_maven_add_param} \
         clean install \
         -Djava.security.egd=file:/dev/./urandomelse
-else        
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        versions:set \
-        -DnewVersion=${bamboo_version_tag}
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        --no-transfer-progress \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-        versions:set  \
-        -DnewVersion=${bamboo_version_tag}
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-        --no-transfer-progress \
-        clean deploy \
-        -e \
-        -DdeployAtEnd=true \
-        -Dmaven.wagon.http.ssl.insecure=true \
-        -Dmaven.wagon.http.ssl.allowall=true \
-        -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-        -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+    echo "dev build with tag ${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
+    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
+    exit 0
 fi
 
-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+echo "gitTag=${newVersion}" > git.tag
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    ${bamboo_maven_add_param} \
+    versions:set \
+    -DnewVersion=${newVersion}
+
+mvn --no-transfer-progress \
+    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    ${bamboo_maven_add_param} \
+    versions:set  \
+    -DnewVersion=${newVersion}
+
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
     --no-transfer-progress \
+    clean deploy \
+    ${bamboo_maven_add_param} \
+    -e \
+    -DdeployAtEnd=true \
+    -Dmaven.wagon.http.ssl.insecure=true \
+    -Dmaven.wagon.http.ssl.allowall=true \
+    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
+    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
+
+mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
     package
 
-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-    --no-transfer-progress \
+mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
     docker:push
-
-if [[ "${bamboo_version_tag}" = "dev" ]]
-then 
-    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
-else
-    echo "gitTag=${bamboo_version_tag}" > git.tag
-fi

bamboo-specs/src/main/resources/scripts/sonar-java.sh

@@ -3,46 +3,42 @@ set -e
 
 SERVICE_NAME=$1
 
-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+echo "build jar binaries"
+mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
     --no-transfer-progress \
     clean install \
     -Djava.security.egd=file:/dev/./urandomelse
 
 echo "dependency-check:aggregate"
-${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-    --no-transfer-progress \
+mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
     org.owasp:dependency-check-maven:aggregate
 
 if [[ -z "${bamboo_repository_pr_key}" ]]
 then
     echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-            --no-transfer-progress \
-            -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-            sonar:sonar \
-            -Dsonar.projectKey=RED_$SERVICE_NAME \
-            -Dsonar.host.url=https://sonarqube.iqser.com \
-            -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-            -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-            -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-            -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-            -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-            
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.branch.name=${bamboo_planRepository_1_branch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
 else
     echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
-    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-            --no-transfer-progress \
-            -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-            sonar:sonar \
-            -Dsonar.projectKey=RED_$SERVICE_NAME \
-            -Dsonar.host.url=https://sonarqube.iqser.com \
-            -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-            -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-            -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-            -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-            -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-            -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-            -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
-fi
+    mvn --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        sonar:sonar \
+        -Dsonar.projectKey=RED_$SERVICE_NAME \
+        -Dsonar.host.url=https://sonarqube.iqser.com \
+        -Dsonar.login=${bamboo_sonarqube_api_token_secret} \
+        -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
+        -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
+        -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
+        -Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
+        -Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
+        -Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
+fi

pom.xml

@@ -7,7 +7,7 @@
 
     <artifactId>redaction-service</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>1.0-SNAPSHOT</version>
+    <version>3.0-SNAPSHOT</version>
 
 
     <packaging>pom</packaging>
@@ -18,4 +18,4 @@
         <module>redaction-service-image-v1</module>
     </modules>
 
-</project>
+</project>

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.6.0</version>
+        <version>1.10.0</version>
         <relativePath />
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.11.0</version>
+                <version>1.15.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>
@@ -59,7 +59,7 @@
                     <groupId>org.sonarsource.scanner.maven</groupId>
                     <artifactId>sonar-maven-plugin</artifactId>
                     <version>3.9.0.2155</version>
-                    </plugin>                    
+                    </plugin>
                 <plugin>
                     <groupId>org.owasp</groupId>
                     <artifactId>dependency-check-maven</artifactId>
@@ -92,7 +92,7 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.7</version>
+                <version>0.8.8</version>
                 <executions>
                     <execution>
                         <id>prepare-agent</id>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.85.0</persistence-service.version>
+        <persistence-service.version>1.156.20</persistence-service.version>
     </properties>
 
     <dependencies>
@@ -25,6 +25,12 @@
             <groupId>com.iqser.red.service</groupId>
             <artifactId>persistence-service-api-v1</artifactId>
             <version>${persistence-service.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>redaction-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
     </dependencies>
 </project>

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -66,6 +66,9 @@ public class RedactionLogEntry {
 
     private Set<String> reference = new HashSet<>();
 
+    @Builder.Default
+    private Set<String> importedRedactionIntersections = new HashSet<>();
+
     public boolean lastChangeIsRemoved() {
         return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
     }

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -20,6 +20,11 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>com.iqser.red.commons</groupId>
+            <artifactId>jackson-commons</artifactId>
+            <version>1.0.0</version>
+        </dependency>
         <dependency>
             <groupId>com.iqser.red.commons</groupId>
             <artifactId>storage-commons</artifactId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedSectionText {
+
+    private int sectionNumber;
+    private String text;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.classification.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SimplifiedText {
+
+    private int numberOfPages;
+    private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,42 +1,30 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.stream.Collectors;
-
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.resources.RedactionResource;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.*;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.stream.Collectors;
 
 @Slf4j
 @RestController
@@ -197,7 +185,7 @@ public class RedactionController implements RedactionResource {
         }
 
         log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        var merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
+        var merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
 
         merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -240,7 +240,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonIgnore
     public Rectangle getRectangle() {
 
-        log.debug("Page: '{}', Word: '{}', Rotation: '{}'", page, toString(), textPositions.get(0).getRotation());
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0).getRotation(), textPositions.get(0).getDir());
 
         float height = getTextHeight();
 
@@ -291,6 +291,14 @@ public class TextPositionSequence implements CharSequence {
             posXInit = textPositions.get(0).getYDirAdj() + 2;
             posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
 
+        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
+            posXInit = textPositions.get(0).getPageWidth() - getX1() +1;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
+                    .getWidthDirAdj() - 3;
+            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
+            posYEnd =  textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() - height + 2;
+
         } else {
             posXEnd = textPositions.get(textPositions.size() - 1)
                     .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,12 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
-
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@@ -15,9 +8,13 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
 
 @Slf4j
 @Service
@@ -36,31 +33,37 @@ public class RedactionMessageReceiver {
         log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
         AnalyzeResult result = null;
 
-        switch (analyzeRequest.getMessageType()) {
+        try {
+            switch (analyzeRequest.getMessageType()) {
 
-            case REANALYSE:
-                result = analyzeService.reanalyze(analyzeRequest);
-                log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case REANALYSE:
+                    result = analyzeService.reanalyze(analyzeRequest);
+                    log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
 
-            case STRUCTURE_ANALYSE:
-                result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-                log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case STRUCTURE_ANALYSE:
+                    result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+                    log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
 
-            case ANALYSE:
-                result = analyzeService.analyze(analyzeRequest);
-                log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case ANALYSE:
+                    result = analyzeService.analyze(analyzeRequest);
+                    log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
 
-            case SURROUNDING_TEXT:
-                result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
-                log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case SURROUNDING_TEXT:
+                    result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
+                    log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
+            }
+
+            result.setMessageType(analyzeRequest.getMessageType());
+            fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
+
+        } catch (Exception e) {
+            log.warn("Failed to process analyze request: {}", analyzeRequest, e);
+            fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         }
-
-        result.setMessageType(analyzeRequest.getMessageType());
-        fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -240,6 +240,17 @@ public class Section {
     }
 
 
+    @WhenCondition
+    public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive){
+
+        var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive);
+
+        var matcher = compiledPattern.matcher(text);
+
+        return matcher.find();
+    }
+
+
     @WhenCondition
     public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) {
 
@@ -248,6 +259,60 @@ public class Section {
         return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value);
     }
 
+    @ThenAction
+    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
+
+        expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null);
+    }
+
+
+    @ThenAction
+    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
+                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
+                              @Argument(ArgumentType.REGEX) String valuePattern) {
+
+        if (StringUtils.isEmpty(prefixPattern)) return;
+
+        var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
+        var compiledPrefixPattern = Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive);
+
+        Set<Entity> expanded = new HashSet<>();
+        for (var entity : entities) {
+
+            if (!entity.getType().equals(type) || entity.getTextBefore() == null) {
+                continue;
+            }
+
+            if (valuePattern != null) {
+                var valueMatcher = compiledValuePattern.matcher(entity.getWord());
+                if (!valueMatcher.matches()) {
+                    continue;
+                }
+            }
+
+            var matcher = compiledPrefixPattern.matcher(entity.getTextBefore());
+
+            while (matcher.find()) {
+                var match = matcher.group(group);
+
+                if (StringUtils.isNotBlank(match)) {
+
+                    if (!StringUtils.endsWith(entity.getTextBefore(), match)) {
+                        log.warn("Rules contain invalid group for expandByPrefixRegEx command; prefixPattern:{}, group{}", prefixPattern, group);
+                        continue;
+                    }
+
+                    var expandedEntities = findEntities(match + entity.getWord(), type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE, false);
+                    expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
+                }
+            }
+        }
+
+        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
+        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
+    }
+
 
     @ThenAction
     public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
@@ -262,13 +327,10 @@ public class Section {
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
                               @Argument(ArgumentType.REGEX) String valuePattern) {
 
-        Pattern compiledValuePattern = null;
+        if (StringUtils.isEmpty(suffixPattern)) return;
 
-        if (valuePattern != null) {
-            compiledValuePattern = Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
-        }
-
-        Pattern compiledPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive);
+        var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
+        var compiledSuffixPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive);
 
         Set<Entity> expanded = new HashSet<>();
         for (Entity entity : entities) {
@@ -278,19 +340,25 @@ public class Section {
             }
 
             if (valuePattern != null) {
-                Matcher valueMatcher = compiledValuePattern.matcher(entity.getWord());
+                var valueMatcher = compiledValuePattern.matcher(entity.getWord());
                 if (!valueMatcher.matches()) {
                     continue;
                 }
             }
 
-            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+            var matcher = compiledSuffixPattern.matcher(entity.getTextAfter());
 
             while (matcher.find()) {
-                String match = matcher.group(group);
+                var match = matcher.group(group);
 
                 if (StringUtils.isNotBlank(match)) {
-                    Set<Entity> expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE, false);
+
+                    if (!StringUtils.startsWith(entity.getTextAfter(), match)) {
+                        log.warn("Rules contain invalid group for expandByRegEx command; suffixPattern:{}, group{}", suffixPattern, group);
+                        continue;
+                    }
+
+                    var expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE, false);
                     expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java

@@ -3,6 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 import lombok.Data;
 
 @Data
@@ -10,6 +13,9 @@ public class ImageServiceResponse {
 
     private String dossierId;
     private String fileId;
+
+    @JsonProperty(value = "imageMetadata")
+    @JsonAlias("data")
     private List<ImageMetadata> data = new ArrayList<>();
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -30,6 +32,8 @@ import com.iqser.red.service.redaction.v1.model.SectionGrid;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedSectionText;
+import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
@@ -104,8 +108,9 @@ public class AnalyzeService {
                         .map(SectionArea::getPage)
                         .collect(Collectors.toSet()), sectionText.getSectionAreas())));
 
-        log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        log.info("Store text, simplified text  and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
 
         return AnalyzeResult.builder()
@@ -310,4 +315,16 @@ public class AnalyzeService {
         }
     }
 
+    private SimplifiedText convert(Text text) {
+
+        List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
+        int numberOfPages = 0;
+
+        if (text != null) {
+            numberOfPages = text.getNumberOfPages();
+            text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText())));
+        }
+        return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -102,7 +102,8 @@ public class DictionaryService {
         try {
             DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
 
-            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId) : dictionaryClient.getAllTypesForDossier(dossierId);
+            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, false)
+                    : dictionaryClient.getAllTypesForDossier(dossierId, false);
             if (CollectionUtils.isNotEmpty(typeResponse)) {
 
                 List<DictionaryModel> dictionary = typeResponse

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import java.util.Iterator;
+import java.util.HashSet;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -24,14 +24,15 @@ public class ImportedRedactionService {
 
 
     public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId, String dossierId, String fileId,
-                                          List<RedactionLogEntry> redactionLogEntries, boolean addImportedRedactions) {
+                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                             boolean addImportedRedactions) {
 
         var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
         if (importedRedactions == null) {
             return redactionLogEntries;
         }
 
-        redactionLogEntries.removeIf(redactionLogEntry -> hasIntersections(redactionLogEntry, importedRedactions));
+        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
 
         if (addImportedRedactions) {
             return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
@@ -42,8 +43,8 @@ public class ImportedRedactionService {
 
 
     private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
-                                                          List<RedactionLogEntry> redactionLogEntries,
-                                                          ImportedRedactions importedRedactions) {
+                                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                                             ImportedRedactions importedRedactions) {
 
         for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
             for (ImportedRedaction importedRedaction : importedRedactionsValues) {
@@ -53,7 +54,7 @@ public class ImportedRedactionService {
                         .imported(true)
                         .redacted(true)
                         .positions(importedRedaction.getPositions())
-                        .color(getColor("imported_redaction", dossierTemplateId))
+                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
                         .build();
 
                 redactionLogEntries.add(redactionLogEntry);
@@ -64,7 +65,7 @@ public class ImportedRedactionService {
     }
 
 
-    private boolean hasIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
+    private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
 
         for (Rectangle rectangle : redactionLogEntry.getPositions()) {
             if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
@@ -72,13 +73,15 @@ public class ImportedRedactionService {
                 for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
                     for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
                         if (intersects(importedRedactionPosition, rectangle)) {
-                            return true;
+                            if(redactionLogEntry.getImportedRedactionIntersections() == null){
+                                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                            }
+                            redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
                         }
                     }
                 }
             }
         }
-        return false;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -65,7 +65,7 @@ public class ManualRedactionSurroundingTextService {
                 var resizeItty = manualRedactions.getResizeRedactions().iterator();
                 while (resizeItty.hasNext()) {
                     var manualResizeRedaction = resizeItty.next();
-                    if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
+                    if (manualResizeRedaction.getValue() != null && sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
                         var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
                         manualResizeRedaction.setTextBefore(surroundingText.getLeft());
                         manualResizeRedaction.setTextAfter(surroundingText.getRight());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -6,6 +6,8 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Manual
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -13,8 +15,9 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
-import java.awt.Color;
+import java.awt.*;
 import java.time.OffsetDateTime;
+import java.util.List;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -27,13 +30,15 @@ public class RedactionLogMergeService {
     private final SectionTextService sectionTextService;
 
 
-    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
-                                              String dossierTemplateId, ManualRedactions manualRedactions,
+    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
                                               Set<Integer> excludedPages, List<Type> types, Colors colors) {
 
+        var skippedImportedRedactions = new HashSet<>();
+
         log.info("Merging Redaction log with manual redactions");
         if (manualRedactions != null) {
 
+
             var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
 
             redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
@@ -46,6 +51,10 @@ public class RedactionLogMergeService {
                         .filter(mr -> entry.getId().equals(mr.getId()))
                         .collect(Collectors.toList()), entry, types, colors);
 
+                if (entry.isImported() && !entry.isRedacted()) {
+                    skippedImportedRedactions.add(entry.getId());
+                }
+
                 entry.setComments(manualRedactions.getComments().get(entry.getId()));
 
                 if (excludedPages != null && !excludedPages.isEmpty()) {
@@ -61,6 +70,12 @@ public class RedactionLogMergeService {
 
         Set<String> processedIds = new HashSet<>();
         redactionLog.getRedactionLogEntry().removeIf(entry -> {
+            if (entry.getImportedRedactionIntersections() != null) {
+                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
+                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) {
+                    return true;
+                }
+            }
             if (processedIds.contains(entry.getId())) {
                 log.info("Duplicate annotation found with id {}", entry.getId());
                 return true;
@@ -367,20 +382,25 @@ public class RedactionLogMergeService {
 
     private float[] getColor(List<Type> types, String type) {
 
-        Optional<Type> foundType = types.stream().filter(t -> t.getType().equals(type)).findFirst();
-        if (foundType.isPresent()){
-            return convertColor(foundType.get().getHexColor());
+        var matchingTypes = getMatchingTypes(types, type);
+        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
+        if (foundAndNotDeletedType.isPresent()) {
+            return convertColor(foundAndNotDeletedType.get().getHexColor());
         }
-        return convertColor("#9398a0");
+        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
+        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0"));
     }
 
 
-    boolean isHint(List<Type> types, String type){
-        Optional<Type> foundType = types.stream().filter(t -> t.getType().equals(type)).findFirst();
-        if (foundType.isPresent()){
-            return foundType.get().isHint();
+    boolean isHint(List<Type> types, String type) {
+
+        var matchingTypes = getMatchingTypes(types, type);
+        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
+        if (foundAndNotDeletedType.isPresent()) {
+            return foundAndNotDeletedType.get().isHint();
         }
-       return false;
+        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
+        return firstDeletedType.map(Type::isHint).orElse(false);
     }
 
 
@@ -390,6 +410,14 @@ public class RedactionLogMergeService {
         return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
     }
 
+    private List<Type> getMatchingTypes(List<Type> types, String type) {
+        return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
+    }
+
+    private boolean isDeletedType(Type type) {
+        return type.getSoftDeletedTime() != null;
+    }
+
 
     @Data
     @AllArgsConstructor

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -62,14 +62,16 @@ public class EntitySearchUtils {
 
        Set<Entity> found = find(inputString, values, type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, asRecommendation ? EntityType.RECOMMENDATION : EntityType.ENTITY);
 
-       if(asRecommendation){
-           Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationValues(), type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_RECOMMENDATION);
+       if (details.getEntityType() == EntityType.RECOMMENDATION) {
+           Set<Entity> falseRecommendations = find(inputString, type.getFalseRecommendationsSearch(), details.withEntityType(EntityType.FALSE_RECOMMENDATION));
            removeFalsePositives(found, falseRecommendations);
-           found.addAll(falseRecommendations);
+           // RED-4799 - false positives are no longer in redaction log
+           // found.addAll(falseRecommendations);
        } else {
-           Set<Entity> falsePositives = find(inputString, type.getFalsePositiveValues(), type.getType(),  headline, sectionNumber, isDictionaryEntry, isDossierDictionary, engine, ignoreMinLength, EntityType.FALSE_POSITIVE);
+           Set<Entity> falsePositives = find(inputString, type.getFalsePositiveSearch(), details.withEntityType(EntityType.FALSE_POSITIVE));
            removeFalsePositives(found, falsePositives);
-           found.addAll(falsePositives);
+           // RED-4799 - false positives are no longer in redaction log
+           // found.addAll(falsePositives);
        }
 
        return found;
@@ -164,9 +166,18 @@ public class EntitySearchUtils {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
+                // skip cross-type false positives
+                if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){
+                    continue;
+                }
+                // skip cross-type false reccommendations
+                if(word.getEntityType() == EntityType.FALSE_RECOMMENDATION && inner.getEntityType() == EntityType.RECOMMENDATION && !inner.getType().equals(word.getType())){
+                    continue;
+                }
+
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
-                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && word.getEntityType().equals(EntityType.ENTITY)) {
+                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                         wordsToRemove.add(word);
                     } else {
                         wordsToRemove.add(inner);
@@ -223,6 +234,16 @@ public class EntitySearchUtils {
 
             if (existing.getType().equals(found.getType())) {
                 existing.getEngines().addAll(found.getEngines());
+                existing.setLegalBasis(found.getLegalBasis());
+                existing.setMatchedRule(found.getMatchedRule());
+                existing.setRedactionReason(found.getRedactionReason());
+                if(existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
+                        || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)){
+                    existing.setEntityType(EntityType.ENTITY);
+                    if(found.isRedaction()){
+                        existing.setRedaction(true);
+                    }
+                }
             } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
                 entities.remove(found);
                 entities.add(found);
@@ -280,18 +301,25 @@ public class EntitySearchUtils {
                 }
                 var existingEntity = existingOptional.get();
                 existingEntity.getEngines().addAll(toAdd.getEngines());
+                existingEntity.setLegalBasis(toAdd.getLegalBasis());
+                existingEntity.setMatchedRule(toAdd.getMatchedRule());
+                existingEntity.setRedactionReason(toAdd.getRedactionReason());
             } else {
                 existing.add(toAdd);
             }
         }
     }
 
-
+    /*
+     * returns true if the found entity overlaps with an existing entity in a way
+     * that neither entity is a subset of the other
+     */
     private boolean overlaps(Set<Entity> existingEntities, Entity found) {
 
         for (Entity existing : existingEntities) {
 
-            if (existing.getStart().equals(found.getStart())) {
+            // skip if either start or end is equal
+            if (existing.getStart().equals(found.getStart()) || existing.getEnd().equals(found.getEnd())) {
                 continue;
             }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.storage;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -41,7 +42,10 @@ public class RedactionStorageService {
     @SneakyThrows
     public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
 
-        storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any));
+        try (var baos = new ByteArrayOutputStream()){
+            objectMapper.writeValue(baos, any);
+            storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), baos);
+        }
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -9,6 +9,8 @@ server:
   port: 8080
 
 spring:
+  main:
+    allow-circular-references: true # FIXME
   profiles:
     active: kubernetes
   rabbitmq:

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -7,18 +7,20 @@ import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.springframework.core.io.InputStreamResource;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.InputStream;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPOutputStream;
 
-public class FileSystemBackedStorageService extends S3StorageService {
+public class FileSystemBackedStorageService implements StorageService{
 
     private final Map<String, File> dataMap = new HashMap<>();
 
     public FileSystemBackedStorageService() {
-        super(null, null);
     }
 
     @SneakyThrows
@@ -33,20 +35,73 @@ public class FileSystemBackedStorageService extends S3StorageService {
 
     }
 
+    @Override
+    public void deleteObject(String objectId) {
+        dataMap.remove(objectId);
+    }
+
+    @Override
+    public boolean objectExists(String objectId) {
+        return dataMap.containsKey(objectId);
+    }
+
+    @Override
+    public void init() {
+
+    }
+
+    public List<String> listPaths(){
+        return new ArrayList<>(dataMap.keySet());
+    }
+
+
+    public List<String> listFilePaths(){
+        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
+    }
+
+
     @SneakyThrows
     @Override
     public void storeObject(String objectId, byte[] data) {
-        File tempFile = File.createTempFile("test", ".tmp");
+        File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
 
-        IOUtils.write(data, new FileOutputStream(tempFile));
+        try(var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.write(data, fileOutputStream);
+        }
 
         dataMap.put(objectId, tempFile);
     }
 
+    @Override
+    @SneakyThrows
+    public void storeObject(String objectId, InputStream stream) {
+        File tempFile = File.createTempFile("test", ".tmp");
+
+        try(var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.copy(stream, fileOutputStream);
+        }
+
+        dataMap.put(objectId, tempFile);
+    }
+
+
+    @Override
+    @SneakyThrows
+    public void storeObject(String objectId, ByteArrayOutputStream stream) {
+
+        File tempFile = File.createTempFile("test", ".tmp");
+        try(var fileOutputStream = new FileOutputStream(tempFile)){
+            stream.writeTo(fileOutputStream);
+        }
+
+        dataMap.put(objectId, tempFile);
+    }
+
+
     public void clearStorage() {
         this.dataMap.forEach((k, v) -> {
             v.delete();
         });
         this.dataMap.clear();
     }
-}
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -188,10 +188,10 @@ public class RedactionIntegrationTest {
         loadTypeForTest();
         loadNerForTest();
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getTypeResponse());
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID)).thenReturn(List.of(Type.builder()
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
                 .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
                 .type(DOSSIER_REDACTIONS)
                 .dossierTemplateId(TEST_DOSSIER_ID)
@@ -208,7 +208,7 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
     }
 
-    
+
     private void mockDictionaryCalls(Long version){
 
         when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
@@ -303,6 +303,8 @@ public class RedactionIntegrationTest {
     @Test
     public void redactionExpansionOverlap() throws IOException {
 
+        // F. Lastname, J. Doe, M. Mustermann
+        // Lastname M., Doe J., Mustermann M.
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/ExpansionTest.pdf");
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
 
@@ -1534,6 +1536,44 @@ public class RedactionIntegrationTest {
         }
     }
 
+    @Test
+    public void testExpandByPrefixRegEx() throws IOException {
+
+        assertThat(dictionary.get(AUTHOR).contains("Robinson"));
+        assertThat(! dictionary.get(AUTHOR).contains("Mrs. Robinson"));
+        assertThat(dictionary.get(AUTHOR).contains("Bojangles"));
+        assertThat(! dictionary.get(AUTHOR).contains("Mr. Bojangles"));
+        assertThat(dictionary.get(AUTHOR).contains("Tambourine Man"));
+        assertThat(! dictionary.get(AUTHOR).contains("Mr. Tambourine Man"));
+
+        String fileName = "files/mr-mrs.pdf";
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        ClassPathResource pdfFileResource = new ClassPathResource(fileName);
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var values = redactionLog.getRedactionLogEntry()
+                .stream()
+                .map(RedactionLogEntry::getValue)
+                .collect(Collectors.toList());
+
+        assertThat(values).contains("Mrs. Robinson");
+        assertThat(values).contains("Mr. Bojangles");
+        assertThat(values).contains("Mr. Tambourine Man");
+    }
 
     @SneakyThrows
     private AnalyzeRequest prepareStorage(InputStream stream) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -145,14 +145,17 @@ public class EntitySearchUtilsTest {
         Set<Entity> foundEntities = new HashSet<>();
         Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
+        foundEntities.add(foundEntities3);
 
         // Act
         Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
 
         // Assert
-        assertThat(result.size()).isEqualTo(0);
+        assertThat(result.size()).isEqualTo(1);
+        assertThat(result.contains(foundEntities2));
 
     }
 
@@ -173,16 +176,21 @@ public class EntitySearchUtilsTest {
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        foundEntities.add(foundEntities1);
-        foundEntities.add(foundEntities2);
+        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        foundEntities.add(foundEntitiesOverlap1);
+        foundEntities.add(foundEntitiesOverlap2);
+        foundEntities.add(foundEntitiesSubset1);
+        foundEntities.add(foundEntitiesSubset2);
 
         // Act
         Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
 
         // Assert
-        assertThat(result.size()).isEqualTo(0);
+        assertThat(result.size()).isEqualTo(2);
+        assertThat(result).containsExactlyInAnyOrder(foundEntitiesSubset1, foundEntitiesSubset2);
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml

@@ -9,6 +9,7 @@ ribbon:
 spring:
   main:
     allow-bean-definition-overriding: true
+    allow-circular-references: true # FIXME
 
 processing.kafkastreams: false
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt

@@ -8620,4 +8620,6 @@ Doe
 M. Mustermann
 F. Lastname
 Mustermann
-Lastname
+Lastname
+Bojangles
+Tambourine Man

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -36,6 +36,23 @@ rule "0: Expand CBI Authors with firstname initials"
         section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
     end
 
+rule "0: Expand CBI_author and PII matches with salutation prefix"
+    when
+        Section((matchesType("CBI_author") || matchesType("PII")) && (
+            searchText.contains("Mr")
+            || searchText.contains("Mrs")
+            || searchText.contains("Ms")
+            || searchText.contains("Miss")
+            || searchText.contains("Sir")
+            || searchText.contains("Madam")
+            || searchText.contains("Madame")
+            || searchText.contains("Mme")
+        ))
+    then
+        section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
+        section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
+    end
+
 
 rule "1: Redacted because Section contains Vertebrate"
     when