RED-4218: Do not perform surrounding text analysis for resize redactions with value = null

Merge in RED/redaction-service from RED-4075-backport to release/3.107.x

* commit 'f4281bf40a6ff15bfeceb6eeacb64bfa6fc49c4f':
  RED-4075: Add legalBasis if Rule Entity overrides Ai Recommendation

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -5,23 +5,27 @@ SERVICE_NAME=$1
 
 if [[ "$bamboo_planRepository_branchName" == "master" ]]
 then
-    branchVersion=$(cat pom.xml | grep -Eo "<version>.*" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|')
-    latestVersion=$( semver $(git tag -l $branchVersion.*) | tail -n1)
+    branchVersion=$(cat pom.xml | grep -Eo "    <version>.*-SNAPSHOT</version>" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|' | tr -d ' ')
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
     newVersion="$(semver $latestVersion -p -i minor)"
+    echo "new release on master with version $newVersion"
 elif [[ "$bamboo_planRepository_branchName" == release* ]]
 then
     branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
-    latestVersion=$( semver $(git tag -l $branchVersion.*) | tail -n1)
+    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
     newVersion="$(semver $latestVersion -p -i patch)"
+    echo "new release on $bamboo_planRepository_branchName with version $newVersion"
 elif [[ "${bamboo_version_tag}" != "dev" ]]
 then
     newVersion="${bamboo_version_tag}"
+    echo "new special version bild with $newVersion"
 else
     mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
         --no-transfer-progress \
         ${bamboo_maven_add_param} \
         clean install \
         -Djava.security.egd=file:/dev/./urandomelse
+    echo "dev build with tag ${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
     echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
     exit 0
 fi
@@ -30,11 +34,13 @@ echo "gitTag=${newVersion}" > git.tag
 
 mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+    ${bamboo_maven_add_param} \
     versions:set \
     -DnewVersion=${newVersion}
 
 mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+    ${bamboo_maven_add_param} \
     versions:set  \
     -DnewVersion=${newVersion}
 
@@ -56,4 +62,3 @@ mvn --no-transfer-progress \
 mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
     docker:push
-

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.6.0</version>
+        <version>1.10.0</version>
         <relativePath />
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -59,7 +59,7 @@
                     <groupId>org.sonarsource.scanner.maven</groupId>
                     <artifactId>sonar-maven-plugin</artifactId>
                     <version>3.9.0.2155</version>
-                    </plugin>                    
+                    </plugin>
                 <plugin>
                     <groupId>org.owasp</groupId>
                     <artifactId>dependency-check-maven</artifactId>
@@ -92,7 +92,7 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.7</version>
+                <version>0.8.8</version>
                 <executions>
                     <execution>
                         <id>prepare-agent</id>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.85.0</persistence-service.version>
+        <persistence-service.version>1.143.0</persistence-service.version>
     </properties>
 
     <dependencies>
@@ -25,6 +25,12 @@
             <groupId>com.iqser.red.service</groupId>
             <artifactId>persistence-service-api-v1</artifactId>
             <version>${persistence-service.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>redaction-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
     </dependencies>
 </project>

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java

@@ -66,6 +66,9 @@ public class RedactionLogEntry {
 
     private Set<String> reference = new HashSet<>();
 
+    @Builder.Default
+    private Set<String> importedRedactionIntersections = new HashSet<>();
+
     public boolean lastChangeIsRemoved() {
         return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,42 +1,30 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.stream.Collectors;
-
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.resources.RedactionResource;
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
+import com.iqser.red.service.redaction.v1.server.redaction.service.*;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-import com.iqser.red.service.redaction.v1.model.SectionArea;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
-import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
-import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
-import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.stream.Collectors;
 
 @Slf4j
 @RestController
@@ -197,7 +185,7 @@ public class RedactionController implements RedactionResource {
         }
 
         log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        var merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
+        var merged = redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
 
         merged.getRedactionLogEntry().removeIf(e -> e.isFalsePositive() && !redactionRequest.isIncludeFalsePositives());
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -240,7 +240,7 @@ public class TextPositionSequence implements CharSequence {
     @JsonIgnore
     public Rectangle getRectangle() {
 
-        log.debug("Page: '{}', Word: '{}', Rotation: '{}'", page, toString(), textPositions.get(0).getRotation());
+        log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0).getRotation(), textPositions.get(0).getDir());
 
         float height = getTextHeight();
 
@@ -291,6 +291,14 @@ public class TextPositionSequence implements CharSequence {
             posXInit = textPositions.get(0).getYDirAdj() + 2;
             posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
 
+        } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
+            posXInit = textPositions.get(0).getPageWidth() - getX1() +1;
+            posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
+                    .getWidthDirAdj() - 3;
+            posYInit = textPositions.get(0).getYDirAdj() - height + 2;
+            posYEnd =  textPositions.get(textPositions.size() - 1)
+                    .getYDirAdj() - height + 2;
+
         } else {
             posXEnd = textPositions.get(textPositions.size() - 1)
                     .getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,12 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
-
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.stereotype.Service;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
@@ -15,9 +8,13 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
 
 @Slf4j
 @Service
@@ -36,31 +33,37 @@ public class RedactionMessageReceiver {
         log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
         AnalyzeResult result = null;
 
-        switch (analyzeRequest.getMessageType()) {
+        try {
+            switch (analyzeRequest.getMessageType()) {
 
-            case REANALYSE:
-                result = analyzeService.reanalyze(analyzeRequest);
-                log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case REANALYSE:
+                    result = analyzeService.reanalyze(analyzeRequest);
+                    log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
 
-            case STRUCTURE_ANALYSE:
-                result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-                log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case STRUCTURE_ANALYSE:
+                    result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+                    log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
 
-            case ANALYSE:
-                result = analyzeService.analyze(analyzeRequest);
-                log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case ANALYSE:
+                    result = analyzeService.analyze(analyzeRequest);
+                    log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
 
-            case SURROUNDING_TEXT:
-                result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
-                log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
-                break;
+                case SURROUNDING_TEXT:
+                    result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
+                    log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
+                    break;
+            }
+
+            result.setMessageType(analyzeRequest.getMessageType());
+            fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
+
+        } catch (Exception e) {
+            log.warn("Failed to process analyze request: {}", analyzeRequest, e);
+            fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         }
-
-        result.setMessageType(analyzeRequest.getMessageType());
-        fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -240,6 +240,17 @@ public class Section {
     }
 
 
+    @WhenCondition
+    public boolean containsRegEx(@Argument(ArgumentType.STRING) String regEx, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive){
+
+        var compiledPattern = Patterns.getCompiledPattern(regEx, patternCaseInsensitive);
+
+        var matcher = compiledPattern.matcher(text);
+
+        return matcher.find();
+    }
+
+
     @WhenCondition
     public boolean rowEquals(@Argument(ArgumentType.STRING) String headerName, @Argument(ArgumentType.STRING) String value) {
 
@@ -261,27 +272,29 @@ public class Section {
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
                               @Argument(ArgumentType.REGEX) String valuePattern) {
 
-        Pattern compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
-        Pattern compiledPrefixPattern = prefixPattern == null ? null : Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive);
+        if (StringUtils.isEmpty(prefixPattern)) return;
+
+        var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
+        var compiledPrefixPattern = Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive);
 
         Set<Entity> expanded = new HashSet<>();
-        for (Entity entity : entities) {
+        for (var entity : entities) {
 
             if (!entity.getType().equals(type) || entity.getTextBefore() == null) {
                 continue;
             }
 
             if (valuePattern != null) {
-                Matcher valueMatcher = compiledValuePattern.matcher(entity.getWord());
+                var valueMatcher = compiledValuePattern.matcher(entity.getWord());
                 if (!valueMatcher.matches()) {
                     continue;
                 }
             }
 
-            Matcher matcher = compiledPrefixPattern.matcher(entity.getTextBefore());
+            var matcher = compiledPrefixPattern.matcher(entity.getTextBefore());
 
             while (matcher.find()) {
-                String match = matcher.group(group);
+                var match = matcher.group(group);
 
                 if (StringUtils.isNotBlank(match)) {
 
@@ -290,7 +303,7 @@ public class Section {
                         continue;
                     }
 
-                    Set<Entity> expandedEntities = findEntities(match + entity.getWord(), type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE, false);
+                    var expandedEntities = findEntities(match + entity.getWord(), type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE, false);
                     expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
                 }
             }
@@ -314,13 +327,10 @@ public class Section {
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
                               @Argument(ArgumentType.REGEX) String valuePattern) {
 
-        Pattern compiledValuePattern = null;
+        if (StringUtils.isEmpty(suffixPattern)) return;
 
-        if (valuePattern != null) {
-            compiledValuePattern = Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
-        }
-
-        Pattern compiledPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive);
+        var compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
+        var compiledSuffixPattern = Patterns.getCompiledPattern(suffixPattern, patternCaseInsensitive);
 
         Set<Entity> expanded = new HashSet<>();
         for (Entity entity : entities) {
@@ -330,16 +340,16 @@ public class Section {
             }
 
             if (valuePattern != null) {
-                Matcher valueMatcher = compiledValuePattern.matcher(entity.getWord());
+                var valueMatcher = compiledValuePattern.matcher(entity.getWord());
                 if (!valueMatcher.matches()) {
                     continue;
                 }
             }
 
-            Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
+            var matcher = compiledSuffixPattern.matcher(entity.getTextAfter());
 
             while (matcher.find()) {
-                String match = matcher.group(group);
+                var match = matcher.group(group);
 
                 if (StringUtils.isNotBlank(match)) {
 
@@ -348,7 +358,7 @@ public class Section {
                         continue;
                     }
 
-                    Set<Entity> expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE, false);
+                    var expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE, false);
                     expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java

@@ -3,6 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 import lombok.Data;
 
 @Data
@@ -10,6 +13,9 @@ public class ImageServiceResponse {
 
     private String dossierId;
     private String fileId;
+
+    @JsonProperty(value = "imageMetadata")
+    @JsonAlias("data")
     private List<ImageMetadata> data = new ArrayList<>();
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -102,7 +102,8 @@ public class DictionaryService {
         try {
             DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
 
-            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId) : dictionaryClient.getAllTypesForDossier(dossierId);
+            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, false)
+                    : dictionaryClient.getAllTypesForDossier(dossierId, false);
             if (CollectionUtils.isNotEmpty(typeResponse)) {
 
                 List<DictionaryModel> dictionary = typeResponse

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import java.util.Iterator;
+import java.util.HashSet;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -24,14 +24,15 @@ public class ImportedRedactionService {
 
 
     public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId, String dossierId, String fileId,
-                                          List<RedactionLogEntry> redactionLogEntries, boolean addImportedRedactions) {
+                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                             boolean addImportedRedactions) {
 
         var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
         if (importedRedactions == null) {
             return redactionLogEntries;
         }
 
-        redactionLogEntries.removeIf(redactionLogEntry -> hasIntersections(redactionLogEntry, importedRedactions));
+        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
 
         if (addImportedRedactions) {
             return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
@@ -42,8 +43,8 @@ public class ImportedRedactionService {
 
 
     private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
-                                                          List<RedactionLogEntry> redactionLogEntries,
-                                                          ImportedRedactions importedRedactions) {
+                                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                                             ImportedRedactions importedRedactions) {
 
         for (List<ImportedRedaction> importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
             for (ImportedRedaction importedRedaction : importedRedactionsValues) {
@@ -53,7 +54,7 @@ public class ImportedRedactionService {
                         .imported(true)
                         .redacted(true)
                         .positions(importedRedaction.getPositions())
-                        .color(getColor("imported_redaction", dossierTemplateId))
+                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
                         .build();
 
                 redactionLogEntries.add(redactionLogEntry);
@@ -64,7 +65,7 @@ public class ImportedRedactionService {
     }
 
 
-    private boolean hasIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
+    private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
 
         for (Rectangle rectangle : redactionLogEntry.getPositions()) {
             if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
@@ -72,13 +73,15 @@ public class ImportedRedactionService {
                 for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
                     for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
                         if (intersects(importedRedactionPosition, rectangle)) {
-                            return true;
+                            if(redactionLogEntry.getImportedRedactionIntersections() == null){
+                                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                            }
+                            redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
                         }
                     }
                 }
             }
         }
-        return false;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -65,7 +65,7 @@ public class ManualRedactionSurroundingTextService {
                 var resizeItty = manualRedactions.getResizeRedactions().iterator();
                 while (resizeItty.hasNext()) {
                     var manualResizeRedaction = resizeItty.next();
-                    if (sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
+                    if (manualResizeRedaction.getValue() != null && sectionContainsEntry(sectionArea, manualResizeRedaction.getPositions())) {
                         var surroundingText = findSurroundingText(sectionText, manualResizeRedaction.getValue(), manualResizeRedaction.getPositions());
                         manualResizeRedaction.setTextBefore(surroundingText.getLeft());
                         manualResizeRedaction.setTextAfter(surroundingText.getRight());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -6,6 +6,8 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.Manual
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.*;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -13,8 +15,9 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 
-import java.awt.Color;
+import java.awt.*;
 import java.time.OffsetDateTime;
+import java.util.List;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -27,13 +30,15 @@ public class RedactionLogMergeService {
     private final SectionTextService sectionTextService;
 
 
-    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
-                                              String dossierTemplateId, ManualRedactions manualRedactions,
+    public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions,
                                               Set<Integer> excludedPages, List<Type> types, Colors colors) {
 
+        var skippedImportedRedactions = new HashSet<>();
+
         log.info("Merging Redaction log with manual redactions");
         if (manualRedactions != null) {
 
+
             var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
 
             redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
@@ -46,6 +51,10 @@ public class RedactionLogMergeService {
                         .filter(mr -> entry.getId().equals(mr.getId()))
                         .collect(Collectors.toList()), entry, types, colors);
 
+                if (entry.isImported() && !entry.isRedacted()) {
+                    skippedImportedRedactions.add(entry.getId());
+                }
+
                 entry.setComments(manualRedactions.getComments().get(entry.getId()));
 
                 if (excludedPages != null && !excludedPages.isEmpty()) {
@@ -61,6 +70,12 @@ public class RedactionLogMergeService {
 
         Set<String> processedIds = new HashSet<>();
         redactionLog.getRedactionLogEntry().removeIf(entry -> {
+            if (entry.getImportedRedactionIntersections() != null) {
+                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
+                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType().equals("ocr")))) {
+                    return true;
+                }
+            }
             if (processedIds.contains(entry.getId())) {
                 log.info("Duplicate annotation found with id {}", entry.getId());
                 return true;
@@ -367,20 +382,25 @@ public class RedactionLogMergeService {
 
     private float[] getColor(List<Type> types, String type) {
 
-        Optional<Type> foundType = types.stream().filter(t -> t.getType().equals(type)).findFirst();
-        if (foundType.isPresent()){
-            return convertColor(foundType.get().getHexColor());
+        var matchingTypes = getMatchingTypes(types, type);
+        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
+        if (foundAndNotDeletedType.isPresent()) {
+            return convertColor(foundAndNotDeletedType.get().getHexColor());
         }
-        return convertColor("#9398a0");
+        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
+        return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor("#9398a0"));
     }
 
 
-    boolean isHint(List<Type> types, String type){
-        Optional<Type> foundType = types.stream().filter(t -> t.getType().equals(type)).findFirst();
-        if (foundType.isPresent()){
-            return foundType.get().isHint();
+    boolean isHint(List<Type> types, String type) {
+
+        var matchingTypes = getMatchingTypes(types, type);
+        Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
+        if (foundAndNotDeletedType.isPresent()) {
+            return foundAndNotDeletedType.get().isHint();
         }
-       return false;
+        Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
+        return firstDeletedType.map(Type::isHint).orElse(false);
     }
 
 
@@ -390,6 +410,14 @@ public class RedactionLogMergeService {
         return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
     }
 
+    private List<Type> getMatchingTypes(List<Type> types, String type) {
+        return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());
+    }
+
+    private boolean isDeletedType(Type type) {
+        return type.getSoftDeletedTime() != null;
+    }
+
 
     @Data
     @AllArgsConstructor

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -166,7 +166,7 @@ public class EntitySearchUtils {
             for (Entity inner : entities) {
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
-                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && word.getEntityType().equals(EntityType.ENTITY)) {
+                    if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) {
                         wordsToRemove.add(word);
                     } else {
                         wordsToRemove.add(inner);
@@ -223,6 +223,16 @@ public class EntitySearchUtils {
 
             if (existing.getType().equals(found.getType())) {
                 existing.getEngines().addAll(found.getEngines());
+                existing.setLegalBasis(found.getLegalBasis());
+                existing.setMatchedRule(found.getMatchedRule());
+                existing.setRedactionReason(found.getRedactionReason());
+                if(existing.getEntityType().equals(EntityType.RECOMMENDATION) && found.getEntityType().equals(EntityType.ENTITY)
+                        || existing.getEntityType().equals(EntityType.ENTITY) && found.getEntityType().equals(EntityType.RECOMMENDATION)){
+                    existing.setEntityType(EntityType.ENTITY);
+                    if(found.isRedaction()){
+                        existing.setRedaction(true);
+                    }
+                }
             } else if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
                 entities.remove(found);
                 entities.add(found);
@@ -280,18 +290,25 @@ public class EntitySearchUtils {
                 }
                 var existingEntity = existingOptional.get();
                 existingEntity.getEngines().addAll(toAdd.getEngines());
+                existingEntity.setLegalBasis(toAdd.getLegalBasis());
+                existingEntity.setMatchedRule(toAdd.getMatchedRule());
+                existingEntity.setRedactionReason(toAdd.getRedactionReason());
             } else {
                 existing.add(toAdd);
             }
         }
     }
 
-
+    /*
+     * returns true if the found entity overlaps with an existing entity in a way
+     * that neither entity is a subset of the other
+     */
     private boolean overlaps(Set<Entity> existingEntities, Entity found) {
 
         for (Entity existing : existingEntities) {
 
-            if (existing.getStart().equals(found.getStart())) {
+            // skip if either start or end is equal
+            if (existing.getStart().equals(found.getStart()) || existing.getEnd().equals(found.getEnd())) {
                 continue;
             }
 

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -9,6 +9,8 @@ server:
   port: 8080
 
 spring:
+  main:
+    allow-circular-references: true # FIXME
   profiles:
     active: kubernetes
   rabbitmq:

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -188,10 +188,10 @@ public class RedactionIntegrationTest {
         loadTypeForTest();
         loadNerForTest();
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getTypeResponse());
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID)).thenReturn(List.of(Type.builder()
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
                 .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
                 .type(DOSSIER_REDACTIONS)
                 .dossierTemplateId(TEST_DOSSIER_ID)
@@ -208,7 +208,7 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
     }
 
-    
+
     private void mockDictionaryCalls(Long version){
 
         when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
@@ -303,6 +303,8 @@ public class RedactionIntegrationTest {
     @Test
     public void redactionExpansionOverlap() throws IOException {
 
+        // F. Lastname, J. Doe, M. Mustermann
+        // Lastname M., Doe J., Mustermann M.
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/ExpansionTest.pdf");
         AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
 
@@ -1534,6 +1536,44 @@ public class RedactionIntegrationTest {
         }
     }
 
+    @Test
+    public void testExpandByPrefixRegEx() throws IOException {
+
+        assertThat(dictionary.get(AUTHOR).contains("Robinson"));
+        assertThat(! dictionary.get(AUTHOR).contains("Mrs. Robinson"));
+        assertThat(dictionary.get(AUTHOR).contains("Bojangles"));
+        assertThat(! dictionary.get(AUTHOR).contains("Mr. Bojangles"));
+        assertThat(dictionary.get(AUTHOR).contains("Tambourine Man"));
+        assertThat(! dictionary.get(AUTHOR).contains("Mr. Tambourine Man"));
+
+        String fileName = "files/mr-mrs.pdf";
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        ClassPathResource pdfFileResource = new ClassPathResource(fileName);
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var values = redactionLog.getRedactionLogEntry()
+                .stream()
+                .map(RedactionLogEntry::getValue)
+                .collect(Collectors.toList());
+
+        assertThat(values).contains("Mrs. Robinson");
+        assertThat(values).contains("Mr. Bojangles");
+        assertThat(values).contains("Mr. Tambourine Man");
+    }
 
     @SneakyThrows
     private AnalyzeRequest prepareStorage(InputStream stream) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -145,14 +145,17 @@ public class EntitySearchUtilsTest {
         Set<Entity> foundEntities = new HashSet<>();
         Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
+        foundEntities.add(foundEntities3);
 
         // Act
         Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
 
         // Assert
-        assertThat(result.size()).isEqualTo(0);
+        assertThat(result.size()).isEqualTo(1);
+        assertThat(result.contains(foundEntities2));
 
     }
 
@@ -173,16 +176,21 @@ public class EntitySearchUtilsTest {
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
-        foundEntities.add(foundEntities1);
-        foundEntities.add(foundEntities2);
+        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        foundEntities.add(foundEntitiesOverlap1);
+        foundEntities.add(foundEntitiesOverlap2);
+        foundEntities.add(foundEntitiesSubset1);
+        foundEntities.add(foundEntitiesSubset2);
 
         // Act
         Set<Entity> result = EntitySearchUtils.findNonOverlappingMatchEntities(existingEntities, foundEntities);
 
         // Assert
-        assertThat(result.size()).isEqualTo(0);
+        assertThat(result.size()).isEqualTo(2);
+        assertThat(result).containsExactlyInAnyOrder(foundEntitiesSubset1, foundEntitiesSubset2);
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml

@@ -9,6 +9,7 @@ ribbon:
 spring:
   main:
     allow-bean-definition-overriding: true
+    allow-circular-references: true # FIXME
 
 processing.kafkastreams: false
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt

@@ -8620,4 +8620,6 @@ Doe
 M. Mustermann
 F. Lastname
 Mustermann
-Lastname
+Lastname
+Bojangles
+Tambourine Man

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -36,6 +36,23 @@ rule "0: Expand CBI Authors with firstname initials"
         section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
     end
 
+rule "0: Expand CBI_author and PII matches with salutation prefix"
+    when
+        Section((matchesType("CBI_author") || matchesType("PII")) && (
+            searchText.contains("Mr")
+            || searchText.contains("Mrs")
+            || searchText.contains("Ms")
+            || searchText.contains("Miss")
+            || searchText.contains("Sir")
+            || searchText.contains("Madam")
+            || searchText.contains("Madame")
+            || searchText.contains("Mme")
+        ))
+    then
+        section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
+        section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
+    end
+
 
 rule "1: Redacted because Section contains Vertebrate"
     when