RED-3553 If image overlaps entity and the entity will be removed

Merge in RED/redaction-service from portsFrom3.2.x to master

* commit 'b13a695c64280a97ece198a18605c0f1b5cf753f':
  RED-3611: Added priority mode that only listens to priority queue
  RED-3620: Fixed remove from dictionary

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -17,7 +17,6 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
 import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
 import com.atlassian.bamboo.specs.api.builders.project.Project;
 import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
-import com.atlassian.bamboo.specs.api.builders.Variable;
 import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
 import com.atlassian.bamboo.specs.builders.task.ScriptTask;
 import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
@@ -84,7 +83,6 @@ public class PlanSpec {
                 project(),
                 SERVICE_NAME, new BambooKey(SERVICE_KEY))
                 .description("Plan created from (enter repository url of your plan)")
-                .variables(new Variable("maven_add_param", ""))
                 .stages(new Stage("Default Stage")
                         .jobs(new Job("Default Job",
                                 new BambooKey("JOB1"))
@@ -96,7 +94,6 @@ public class PlanSpec {
                                                         "rm -rf ./*"),
                                         new VcsCheckoutTask()
                                                 .description("Checkout Default Repository")
-                                                .cleanCheckout(true)
                                                 .checkoutItems(new CheckoutItem().defaultRepository()),
                                         new ScriptTask()
                                                 .description("Build")
@@ -118,8 +115,8 @@ public class PlanSpec {
                                                 .defaultRepository())
                                 .dockerConfiguration(
                                         new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
+                                                .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                                 .volume("/var/run/docker.sock", "/var/run/docker.sock")
                                 )
                         )
@@ -150,7 +147,6 @@ public class PlanSpec {
                                                         "rm -rf ./*"),
                                         new VcsCheckoutTask()
                                                 .description("Checkout Default Repository")
-                                                .cleanCheckout(true)
                                                 .checkoutItems(new CheckoutItem().defaultRepository()),
                                         new ScriptTask()
                                                 .description("Sonar")
@@ -159,9 +155,9 @@ public class PlanSpec {
                                                 .argument(SERVICE_NAME))
                                 .dockerConfiguration(
                                         new DockerConfiguration()
-                                                .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
-                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
+                                                .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
                                                 .dockerRunArguments("--net=host")
+                                                .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                                 .volume("/var/run/docker.sock", "/var/run/docker.sock")
                                 )
                         )
@@ -169,7 +165,8 @@ public class PlanSpec {
                 .linkedRepositories("RED / " + SERVICE_NAME)
                 .triggers(
                         new ScheduledTrigger()
-                                .scheduleOnceDaily(LocalTime.of(23, 00)))
+                                .scheduleOnceDaily(LocalTime.of(12, 00)),
+                        new BitbucketServerTrigger())
                 .planBranchManagement(new PlanBranchManagement()
                         .createForVcsBranchMatching("release.*")
                         .notificationForCommitters());

bamboo-specs/src/main/resources/scripts/build-java.sh

@@ -3,56 +3,49 @@ set -e
 
 SERVICE_NAME=$1
 
-if [[ "$bamboo_planRepository_branchName" == "master" ]]
-then
-    branchVersion=$(cat pom.xml | grep -Eo "<version>.*" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|')
-    latestVersion=$( semver $(git tag -l $branchVersion.*) | tail -n1)
-    newVersion="$(semver $latestVersion -p -i minor)"
-elif [[ "$bamboo_planRepository_branchName" == release* ]]
-then
-    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
-    latestVersion=$( semver $(git tag -l $branchVersion.*) | tail -n1)
-    newVersion="$(semver $latestVersion -p -i patch)"
-elif [[ "${bamboo_version_tag}" != "dev" ]]
-then
-    newVersion="${bamboo_version_tag}"
-else
-    mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+if [[ "${bamboo_version_tag}" = "dev" ]]
+then 
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
         --no-transfer-progress \
-        ${bamboo_maven_add_param} \
         clean install \
         -Djava.security.egd=file:/dev/./urandomelse
-    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
-    exit 0
+else        
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        versions:set \
+        -DnewVersion=${bamboo_version_tag}
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        --no-transfer-progress \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
+        versions:set  \
+        -DnewVersion=${bamboo_version_tag}
+    ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+        -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+        --no-transfer-progress \
+        clean deploy \
+        -e \
+        -DdeployAtEnd=true \
+        -Dmaven.wagon.http.ssl.insecure=true \
+        -Dmaven.wagon.http.ssl.allowall=true \
+        -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
+        -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
 fi
 
-echo "gitTag=${newVersion}" > git.tag
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
-    versions:set \
-    -DnewVersion=${newVersion}
-
-mvn --no-transfer-progress \
-    -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
-    versions:set  \
-    -DnewVersion=${newVersion}
-
-mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
+${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
     --no-transfer-progress \
-    clean deploy \
-    ${bamboo_maven_add_param} \
-    -e \
-    -DdeployAtEnd=true \
-    -Dmaven.wagon.http.ssl.insecure=true \
-    -Dmaven.wagon.http.ssl.allowall=true \
-    -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-    -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
-
-mvn --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
     package
 
-mvn --no-transfer-progress \
+${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
+    --no-transfer-progress \
     -f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
     docker:push
+
+if [[ "${bamboo_version_tag}" = "dev" ]]
+then 
+    echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
+else
+    echo "gitTag=${bamboo_version_tag}" > git.tag
+fi

pom.xml

@@ -7,7 +7,7 @@
 
     <artifactId>redaction-service</artifactId>
     <groupId>com.iqser.red.service</groupId>
-    <version>3.8-SNAPSHOT</version>
+    <version>1.0-SNAPSHOT</version>
 
 
     <packaging>pom</packaging>

redaction-service-v1/redaction-service-api-v1/pom.xml

@@ -12,7 +12,7 @@
     <artifactId>redaction-service-api-v1</artifactId>
 
     <properties>
-        <persistence-service.version>1.70.2</persistence-service.version>
+        <persistence-service.version>1.85.0</persistence-service.version>
     </properties>
 
     <dependencies>

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java

@@ -20,7 +20,6 @@ public class ManualChange {
     private AnnotationStatus annotationStatus;
     private ManualRedactionType manualRedactionType;
     private OffsetDateTime processedDate;
-    private OffsetDateTime requestedDate;
     private String userId;
     private Map<String, Object> propertyChanges = new HashMap<>();
 
@@ -32,7 +31,6 @@ public class ManualChange {
         ManualChange manualChange = new ManualChange();
         manualChange.annotationStatus = baseAnnotation.getStatus();
         manualChange.processedDate = baseAnnotation.getProcessedDate();
-        manualChange.requestedDate = baseAnnotation.getRequestDate();
         manualChange.userId = baseAnnotation.getUser();
         return manualChange;
     }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java

@@ -1,14 +1,10 @@
 package com.iqser.red.service.redaction.v1.model;
 
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 
 
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -26,7 +22,4 @@ public class RedactionRequest {
     private ManualRedactions manualRedactions;
     @Builder.Default
     private Set<Integer> excludedPages = new HashSet<>();
-
-    private Colors colors;
-    private List<Type> types;
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -1,19 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;
 
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.regex.Pattern;
 
 @Slf4j
 @Service
@@ -54,7 +52,7 @@ public class ClassificationService {
                               List<Float> headlineFontSizes) {
 
         if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification("Other");
+            // TODO Figure out why this happens.
             return;
         }
         if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -170,6 +170,7 @@ public class RedactionController implements RedactionResource {
     public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
 
         log.debug("Requested preview for: {}", redactionRequest);
+        dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());
 
         var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
 
@@ -197,7 +198,7 @@ public class RedactionController implements RedactionResource {
         }
 
         log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion());
-        return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages(), redactionRequest.getTypes(), redactionRequest.getColors());
+        return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -249,58 +249,6 @@ public class Section {
         return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value);
     }
 
-    @ThenAction
-    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) {
-
-        expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null);
-    }
-
-
-    @ThenAction
-    public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group,
-                              @Argument(ArgumentType.REGEX) String valuePattern) {
-
-        Pattern compiledValuePattern = valuePattern == null ? null : Patterns.getCompiledPattern(valuePattern, patternCaseInsensitive);
-        Pattern compiledPrefixPattern = prefixPattern == null ? null : Patterns.getCompiledPattern(prefixPattern, patternCaseInsensitive);
-
-        Set<Entity> expanded = new HashSet<>();
-        for (Entity entity : entities) {
-
-            if (!entity.getType().equals(type) || entity.getTextBefore() == null) {
-                continue;
-            }
-
-            if (valuePattern != null) {
-                Matcher valueMatcher = compiledValuePattern.matcher(entity.getWord());
-                if (!valueMatcher.matches()) {
-                    continue;
-                }
-            }
-
-            Matcher matcher = compiledPrefixPattern.matcher(entity.getTextBefore());
-
-            while (matcher.find()) {
-                String match = matcher.group(group);
-
-                if (StringUtils.isNotBlank(match)) {
-
-                    if (!StringUtils.endsWith(entity.getTextBefore(), match)) {
-                        log.warn("Rules contain invalid group for expandByPrefixRegEx command; prefixPattern:{}, group{}", prefixPattern, group);
-                        continue;
-                    }
-
-                    Set<Entity> expandedEntities = findEntities(match + entity.getWord(), type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE);
-                    expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
-                }
-            }
-        }
-
-        EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
-        EntitySearchUtils.removeEntitiesContainedInLarger(entities);
-    }
-
 
     @ThenAction
     public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
@@ -343,12 +291,6 @@ public class Section {
                 String match = matcher.group(group);
 
                 if (StringUtils.isNotBlank(match)) {
-
-                    if (!StringUtils.startsWith(entity.getTextAfter(), match)) {
-                        log.warn("Rules contain invalid group for expandByRegEx command; suffixPattern:{}, group{}", suffixPattern, group);
-                        continue;
-                    }
-
                     Set<Entity> expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE);
                     expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
                 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -61,6 +61,8 @@ public class EntityRedactionService {
         }
 
         Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(entities);
+        EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
+
         return new PageEntities(entitiesPerPage, imagesPerPage);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java

@@ -31,7 +31,7 @@ public class ImportedRedactionService {
             return redactionLogEntries;
         }
 
-        redactionLogEntries.removeIf(redactionLogEntry -> !redactionLogEntry.isImage() && hasIntersections(redactionLogEntry, importedRedactions));
+        redactionLogEntries.removeIf(redactionLogEntry -> hasIntersections(redactionLogEntry, importedRedactions));
 
         if (addImportedRedactions) {
             return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -1,80 +1,38 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import java.awt.Color;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.ManualChange;
-import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
-import com.iqser.red.service.redaction.v1.model.Point;
-import com.iqser.red.service.redaction.v1.model.Rectangle;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.SectionGrid;
-
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
+import com.iqser.red.service.redaction.v1.model.*;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.time.OffsetDateTime;
+import java.util.*;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class RedactionLogMergeService {
 
+    private final DictionaryService dictionaryService;
 
     private final SectionTextService sectionTextService;
 
 
     public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid,
                                               String dossierTemplateId, ManualRedactions manualRedactions,
-                                              Set<Integer> excludedPages, List<Type> types, Colors colors) {
+                                              Set<Integer> excludedPages) {
 
         log.info("Merging Redaction log with manual redactions");
         if (manualRedactions != null) {
 
-            Map<String, OffsetDateTime> idsToRemove = manualRedactions.getIdsToRemove()
-                    .stream()
-                    .collect(Collectors.toMap(BaseAnnotation::getAnnotationId, BaseAnnotation::getRequestDate));
-
-            List<ManualRedactionEntry> entriesToRemove = new ArrayList<>();
-            Set<String> removesToRemove = new HashSet<>();
-
-            manualRedactions.getEntriesToAdd().forEach(m -> {
-                if (idsToRemove.containsKey(m.getAnnotationId()) && idsToRemove.get(m.getAnnotationId()).isAfter(m.getRequestDate())) {
-                    entriesToRemove.add(m);
-                } else if (idsToRemove.containsKey(m.getAnnotationId()) && idsToRemove.get(m.getAnnotationId()).isBefore(m.getRequestDate())) {
-                    removesToRemove.add(m.getAnnotationId());
-                }
-            });
-
-            log.debug("Will remove these manual redactions {}", entriesToRemove);
-            log.debug("Will remove these id removals {}", removesToRemove);
-            manualRedactions.getIdsToRemove().removeIf(r -> removesToRemove.contains(r.getAnnotationId()));
-            manualRedactions.getEntriesToAdd().removeAll(entriesToRemove);
-
-            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), colors, types);
+            var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);
 
             redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
 
@@ -84,7 +42,7 @@ public class RedactionLogMergeService {
 
                 processRedactionLogEntry(manualRedactionWrappers.stream()
                         .filter(mr -> entry.getId().equals(mr.getId()))
-                        .collect(Collectors.toList()), entry, types, colors);
+                        .collect(Collectors.toList()), dossierTemplateId, entry);
 
                 entry.setComments(manualRedactions.getComments().get(entry.getId()));
 
@@ -153,7 +111,7 @@ public class RedactionLogMergeService {
 
 
     private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers,
-                                          RedactionLogEntry redactionLogEntry, List<Type> types, Colors colors) {
+                                          String dossierTemplateId, RedactionLogEntry redactionLogEntry) {
 
         manualRedactionWrappers.forEach(mrw -> {
 
@@ -166,7 +124,7 @@ public class RedactionLogMergeService {
                     redactionLogEntry.setType(imageRecategorization.getType());
                     redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
 
-                    if (isHint(types, imageRecategorization.getType())) {
+                    if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
                         redactionLogEntry.setRedacted(false);
                         redactionLogEntry.setHint(true);
                     } else {
@@ -176,7 +134,7 @@ public class RedactionLogMergeService {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
                 } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
                 }
 
                 redactionLogEntry.setReason(manualOverrideReason);
@@ -195,11 +153,11 @@ public class RedactionLogMergeService {
                     if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
                         redactionLogEntry.setRedacted(false);
                         manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), true, types));
+                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
                         redactionLogEntry.setHint(false);
                     } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
                         manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
+                        redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
                     }
 
                     redactionLogEntry.setReason(manualOverrideReason);
@@ -213,7 +171,7 @@ public class RedactionLogMergeService {
             if (mrw.getItem() instanceof ManualForceRedaction) {
                 var manualForceRedact = (ManualForceRedaction) mrw.getItem();
                 String manualOverrideReason = null;
-                var dictionaryIsHint = isHint(types, redactionLogEntry.getType());
+                var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
                 if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
                     // Forcing a skipped hint should result in a hint
                     if (dictionaryIsHint) {
@@ -221,12 +179,12 @@ public class RedactionLogMergeService {
                     } else {
                         redactionLogEntry.setRedacted(true);
                     }
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
                     redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                 } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
                     redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
                 }
 
@@ -253,7 +211,7 @@ public class RedactionLogMergeService {
                     }
                 } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
                 }
 
                 redactionLogEntry.setReason(manualOverrideReason);
@@ -274,7 +232,7 @@ public class RedactionLogMergeService {
                 var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
                 String manualOverrideReason = null;
                 if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, false, redactionLogEntry.isRedacted(), false, types));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
                     redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                     redactionLogEntry.setValue(manualResizeRedact.getValue());
                     redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
@@ -282,7 +240,7 @@ public class RedactionLogMergeService {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
                 } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), colors, true, redactionLogEntry.isRedacted(), false, types));
+                    redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
                     redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
                     redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
                     redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
@@ -313,14 +271,14 @@ public class RedactionLogMergeService {
 
 
     public List<RedactionLogEntry> addManualAddEntries(SectionGrid sectionGrid, Set<ManualRedactionEntry> manualAdds,
-                                                       Map<String, List<Comment>> comments, Colors colors, List<Type> types) {
+                                                       Map<String, List<Comment>> comments, String dossierTemplateId) {
 
         List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
 
         for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
 
-            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types);
+            if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
+                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
                 redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
                 redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
                 redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
@@ -344,15 +302,16 @@ public class RedactionLogMergeService {
                 .collect(Collectors.toList());
     }
 
-    @SuppressWarnings("PMD.UselessParentheses")
-    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary())
-                || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDictionary()) && manualRedactionEntry.getProcessedDate() == null);
+
+    private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
+
+        return manualRedactionEntry.getStatus().equals(AnnotationStatus.APPROVED) &&
+                (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
     }
 
 
     private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
-                                                      Colors colors, List<Type> types) {
+                                                      String dossierTemplateId) {
 
         var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
 
@@ -363,7 +322,7 @@ public class RedactionLogMergeService {
 
         return RedactionLogEntry.builder()
                 .id(id)
-                .color(getColorForManualAdd(manualRedactionEntry.getType(), colors, manualRedactionEntry.getStatus(), types))
+                .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
                 .reason(manualRedactionEntry.getReason())
                 .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
                 .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
@@ -380,53 +339,33 @@ public class RedactionLogMergeService {
     }
 
 
-    private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction,
-                             boolean skipped, List<Type> types) {
+    private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction,
+                             boolean skipped) {
 
         if (requested) {
-            return convertColor(colors.getRequestRemove());
+            return dictionaryService.getRequestRemoveColor(dossierTemplateId);
         }
-        if (skipped || !isRedaction && !isHint(types, type)) {
-            return convertColor(colors.getNotRedacted());
+        if (skipped || !isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
         }
-        return getColor(types, type);
+        return dictionaryService.getColor(type, dossierTemplateId);
     }
 
 
-    private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
+    private float[] getColorForManualAdd(String type, String dossierTemplateId, AnnotationStatus status) {
 
         if (status.equals(AnnotationStatus.REQUESTED)) {
-            return convertColor(colors.getRequestAdd());
+            return dictionaryService.getRequestAddColor(dossierTemplateId);
         } else if (status.equals(AnnotationStatus.DECLINED)) {
-            return convertColor(colors.getNotRedacted());
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
         }
-        return getColor(types, type);
+        return getColor(type, dossierTemplateId);
     }
 
 
-    private float[] getColor(List<Type> types, String type) {
+    private float[] getColor(String type, String dossierTemplateId) {
 
-        Optional<Type> foundType = types.stream().filter(t -> t.getType().equals(type)).findFirst();
-        if (foundType.isPresent()){
-            return convertColor(foundType.get().getHexColor());
-        }
-        return convertColor("#9398a0");
-    }
-
-
-    boolean isHint(List<Type> types, String type){
-        Optional<Type> foundType = types.stream().filter(t -> t.getType().equals(type)).findFirst();
-        if (foundType.isPresent()){
-            return foundType.get().isHint();
-        }
-       return false;
-    }
-
-
-    private float[] convertColor(String hex) {
-
-        Color color = Color.decode(hex);
-        return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
+        return dictionaryService.getColor(type, dossierTemplateId);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.experimental.UtilityClass;
@@ -54,8 +55,8 @@ public class EntitySearchUtils {
     }
 
 
-    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                            boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine, boolean ignoreMinLength) {
+    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionary,
+                            Engine engine, boolean ignoreMinLength) {
 
         Set<Entity> found = new HashSet<>();
 
@@ -98,13 +99,10 @@ public class EntitySearchUtils {
 
         for (String word : entitiesByWord.keySet()) {
 
-            List<Entity> orderedEntities = entitiesByWord.get(word)
-                    .stream()
-                    .sorted(Comparator.comparing(Entity::getStart))
-                    .collect(Collectors.toList());
+            List<Entity> orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
             Entity firstEntity = orderedEntities.get(0);
             List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord()
-                    .trim(), dictionary == null ? true : dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
+                    .trim(), dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
 
             for (int i = 0; i <= orderedEntities.size() - 1; i++) {
                 try {
@@ -129,7 +127,7 @@ public class EntitySearchUtils {
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     // FIXME this is workaround for RED-3327 and should be removed in the future.
-                    if(word.getType().contains("recommendation_") && !inner.getType().contains("recommendation_")) {
+                    if (word.getType().contains("recommendation_") && !inner.getType().contains("recommendation_")) {
                         wordsToRemove.add(word);
                     } else {
                         wordsToRemove.add(inner);
@@ -137,7 +135,35 @@ public class EntitySearchUtils {
                 }
             }
         }
-        entities.removeAll(wordsToRemove);
+        wordsToRemove.forEach(entities::remove);
+    }
+
+
+    public static void removeEntitiesContainedInRedactedLogos(Map<Integer, Set<Image>> imagesPerPage, Map<Integer, List<Entity>> entitiesPerPage) {
+
+        if (entitiesPerPage != null && imagesPerPage != null) {
+            for (int page : imagesPerPage.keySet()) {
+                Set<Entity> removeEntitiesFromPage = new HashSet<>();
+
+                if (!entitiesPerPage.containsKey(page)) {
+                    continue;
+                }
+                imagesPerPage.get(page).forEach(image -> {
+                    if (image.isRedaction()) {
+                        entitiesPerPage.get(page).forEach(entity -> {
+                            if (isImageOverlappingEntity(image, entity)) {
+                                log.info("Logo overlaps entity and entity will be removed");
+                                removeEntitiesFromPage.add(entity);
+                            }
+
+                        });
+                    }
+                });
+
+                entitiesPerPage.get(page).removeAll(removeEntitiesFromPage);
+
+            }
+        }
     }
 
 
@@ -175,22 +201,17 @@ public class EntitySearchUtils {
     }
 
 
-    public void addOrAddEngine(Set<Entity> existing, Set<Entity> toBeAdded) {
+    private boolean isImageOverlappingEntity(Image image, Entity entity) {
 
-        for (Entity toAdd : toBeAdded) {
-            if (existing.contains(toAdd)) {
-                Optional<Entity> existingOptional = existing.stream()
-                        .filter(entity -> entity.equals(toAdd))
-                        .findFirst();
-                if (!existingOptional.isPresent()) {
-                    return;
-                }
-                var existingEntity = existingOptional.get();
-                existingEntity.getEngines().addAll(toAdd.getEngines());
-            } else {
-                existing.add(toAdd);
-            }
-        }
+        return image.getPosition() != null && entity.getPositionSequences() != null && entity.getPositionSequences().get(0) != null && entity.getPositionSequences()
+                .get(0)
+                .getSequences() != null && entity.getPositionSequences().get(0).getSequences().get(0) != null && image.getPosition().getX() < entity.getPositionSequences()
+                .get(0)
+                .getSequences()
+                .get(0)
+                .getX1() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences().get(0).getSequences().get(0).getX2() && image.getPosition()
+                .getY() < entity.getPositionSequences().get(0).getSequences().get(0).getY1() && image.getPosition().getY() + image.getPosition()
+                .getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getY2();
     }
 
 
@@ -210,11 +231,28 @@ public class EntitySearchUtils {
     }
 
 
+    public void addOrAddEngine(Set<Entity> existing, Set<Entity> toBeAdded) {
+
+        for (Entity toAdd : toBeAdded) {
+            if (existing.contains(toAdd)) {
+                Optional<Entity> existingOptional = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst();
+                if (!existingOptional.isPresent()) {
+                    return;
+                }
+                var existingEntity = existingOptional.get();
+                existingEntity.getEngines().addAll(toAdd.getEngines());
+            } else {
+                existing.add(toAdd);
+            }
+        }
+    }
+
+
     private boolean overlaps(Set<Entity> existingEntities, Entity found) {
 
         for (Entity existing : existingEntities) {
 
-            if(existing.getStart().equals(found.getStart())){
+            if (existing.getStart().equals(found.getStart())) {
                 continue;
             }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -328,7 +328,6 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
     @SneakyThrows
     public void testIgnoreHint() {
 
@@ -972,7 +971,6 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
     public void resizeRedactionTest() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
@@ -1128,7 +1126,6 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
     public void testManualSurroundingText() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/new/S4.pdf");