RED-3553 If image overlaps entity and the entity will be removed

Merge in RED/redaction-service from portsFrom3.2.x to master

* commit 'b13a695c64280a97ece198a18605c0f1b5cf753f':
  RED-3611: Added priority mode that only listens to priority queue
  RED-3620: Fixed remove from dictionary

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class MessageReceiver {
+
+    private final RedactionMessageReceiver redactionMessageReceiver;
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_QUEUE)
+    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+        redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
 import lombok.RequiredArgsConstructor;
+
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -10,11 +12,24 @@ import org.springframework.context.annotation.Configuration;
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
-
     public static final String REDACTION_QUEUE = "redactionQueue";
-
     public static final String REDACTION_DQL = "redactionDQL";
 
+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
+    public MessageReceiver messageReceiver(RedactionMessageReceiver redactionMessageReceiver){
+        return new MessageReceiver(redactionMessageReceiver);
+    }
+
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
+    public PriorityMessageReceiver priorityMessageReceiver(RedactionMessageReceiver redactionMessageReceiver){
+        return new PriorityMessageReceiver(redactionMessageReceiver);
+    }
+
 
     @Bean
     public Queue redactionQueue() {
@@ -27,9 +42,21 @@ public class MessagingConfiguration {
     }
 
 
+    @Bean
+    public Queue redactionPriorityQueue() {
+
+        return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
+                .maxPriority(2)
+                .build();
+    }
+
+
     @Bean
     public Queue redactionDeadLetterQueue() {
 
         return QueueBuilder.durable(REDACTION_DQL).build();
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class PriorityMessageReceiver {
+
+    private final RedactionMessageReceiver redactionMessageReceiver;
+
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_PRIORITY_QUEUE)
+    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+
+        redactionMessageReceiver.receiveAnalyzeRequest(in, true);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -30,12 +30,10 @@ public class RedactionMessageReceiver {
     private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
 
 
-    @RabbitHandler
-    @RabbitListener(queues = REDACTION_QUEUE)
-    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+    public void receiveAnalyzeRequest(String in, boolean priority) throws JsonProcessingException {
 
         var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
-        log.info("Processing analyze request for file: {}", analyzeRequest.getFileId());
+        log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
         AnalyzeResult result = null;
 
         switch (analyzeRequest.getMessageType()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -99,8 +99,13 @@ public class DictionaryService {
                             }
 
                             Set<DictionaryEntry> entries = new HashSet<>();
+                            var newEntries = convertEntries(t.getId(), currentVersion);
+                            var newValues = newEntries.stream().map(v -> v.getValue()).collect(Collectors.toSet());
+
                             // add old entries from existing DictionaryModel
-                            oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries()));
+                            oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(
+                                    f -> !newValues.contains(f.getValue())).collect(Collectors.toList())
+                            ));
                             // Add Increments
                             entries.addAll(convertEntries(t.getId(), currentVersion));
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -61,6 +61,8 @@ public class EntityRedactionService {
         }
 
         Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(entities);
+        EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
+
         return new PageEntities(entitiesPerPage, imagesPerPage);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.experimental.UtilityClass;
@@ -54,8 +55,8 @@ public class EntitySearchUtils {
     }
 
 
-    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
-                            boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine, boolean ignoreMinLength) {
+    public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionary,
+                            Engine engine, boolean ignoreMinLength) {
 
         Set<Entity> found = new HashSet<>();
 
@@ -98,13 +99,10 @@ public class EntitySearchUtils {
 
         for (String word : entitiesByWord.keySet()) {
 
-            List<Entity> orderedEntities = entitiesByWord.get(word)
-                    .stream()
-                    .sorted(Comparator.comparing(Entity::getStart))
-                    .collect(Collectors.toList());
+            List<Entity> orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
             Entity firstEntity = orderedEntities.get(0);
             List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord()
-                    .trim(), dictionary == null ? true : dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
+                    .trim(), dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()), firstEntity.getTargetSequences());
 
             for (int i = 0; i <= orderedEntities.size() - 1; i++) {
                 try {
@@ -129,7 +127,7 @@ public class EntitySearchUtils {
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
                     // FIXME this is workaround for RED-3327 and should be removed in the future.
-                    if(word.getType().contains("recommendation_") && !inner.getType().contains("recommendation_")) {
+                    if (word.getType().contains("recommendation_") && !inner.getType().contains("recommendation_")) {
                         wordsToRemove.add(word);
                     } else {
                         wordsToRemove.add(inner);
@@ -137,7 +135,35 @@ public class EntitySearchUtils {
                 }
             }
         }
-        entities.removeAll(wordsToRemove);
+        wordsToRemove.forEach(entities::remove);
+    }
+
+
+    public static void removeEntitiesContainedInRedactedLogos(Map<Integer, Set<Image>> imagesPerPage, Map<Integer, List<Entity>> entitiesPerPage) {
+
+        if (entitiesPerPage != null && imagesPerPage != null) {
+            for (int page : imagesPerPage.keySet()) {
+                Set<Entity> removeEntitiesFromPage = new HashSet<>();
+
+                if (!entitiesPerPage.containsKey(page)) {
+                    continue;
+                }
+                imagesPerPage.get(page).forEach(image -> {
+                    if (image.isRedaction()) {
+                        entitiesPerPage.get(page).forEach(entity -> {
+                            if (isImageOverlappingEntity(image, entity)) {
+                                log.info("Logo overlaps entity and entity will be removed");
+                                removeEntitiesFromPage.add(entity);
+                            }
+
+                        });
+                    }
+                });
+
+                entitiesPerPage.get(page).removeAll(removeEntitiesFromPage);
+
+            }
+        }
     }
 
 
@@ -175,22 +201,17 @@ public class EntitySearchUtils {
     }
 
 
-    public void addOrAddEngine(Set<Entity> existing, Set<Entity> toBeAdded) {
+    private boolean isImageOverlappingEntity(Image image, Entity entity) {
 
-        for (Entity toAdd : toBeAdded) {
-            if (existing.contains(toAdd)) {
-                Optional<Entity> existingOptional = existing.stream()
-                        .filter(entity -> entity.equals(toAdd))
-                        .findFirst();
-                if (!existingOptional.isPresent()) {
-                    return;
-                }
-                var existingEntity = existingOptional.get();
-                existingEntity.getEngines().addAll(toAdd.getEngines());
-            } else {
-                existing.add(toAdd);
-            }
-        }
+        return image.getPosition() != null && entity.getPositionSequences() != null && entity.getPositionSequences().get(0) != null && entity.getPositionSequences()
+                .get(0)
+                .getSequences() != null && entity.getPositionSequences().get(0).getSequences().get(0) != null && image.getPosition().getX() < entity.getPositionSequences()
+                .get(0)
+                .getSequences()
+                .get(0)
+                .getX1() && image.getPosition().getX() + image.getPosition().getWidth() > entity.getPositionSequences().get(0).getSequences().get(0).getX2() && image.getPosition()
+                .getY() < entity.getPositionSequences().get(0).getSequences().get(0).getY1() && image.getPosition().getY() + image.getPosition()
+                .getHeight() > entity.getPositionSequences().get(0).getSequences().get(0).getY2();
     }
 
 
@@ -210,11 +231,28 @@ public class EntitySearchUtils {
     }
 
 
+    public void addOrAddEngine(Set<Entity> existing, Set<Entity> toBeAdded) {
+
+        for (Entity toAdd : toBeAdded) {
+            if (existing.contains(toAdd)) {
+                Optional<Entity> existingOptional = existing.stream().filter(entity -> entity.equals(toAdd)).findFirst();
+                if (!existingOptional.isPresent()) {
+                    return;
+                }
+                var existingEntity = existingOptional.get();
+                existingEntity.getEngines().addAll(toAdd.getEngines());
+            } else {
+                existing.add(toAdd);
+            }
+        }
+    }
+
+
     private boolean overlaps(Set<Entity> existingEntities, Entity found) {
 
         for (Entity existing : existingEntities) {
 
-            if(existing.getStart().equals(found.getStart())){
+            if (existing.getStart().equals(found.getStart())) {
                 continue;
             }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java

@@ -20,4 +20,6 @@ public class RedactionServiceSettings {
 
     private boolean nerServiceEnabled = true;
 
+    private boolean priorityMode;
+
 }