Pull request #338: RED-3327: Ignore recommendations if touches other type

Merge in RED/redaction-service from RED-3327 to release/3.54.x

* commit '07ffd900582ada731a7381d825509ed5e656b70e':
  RED-3327: Ignore recommendations if touches other type

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java

@@ -26,6 +26,7 @@ public class AnalyzeRequest {
     private String dossierTemplateId;
     private ManualRedactions manualRedactions;
     private OffsetDateTime lastProcessed;
+    private int analysisNumber;
 
     @Builder.Default
     private Set<Integer> excludedPages = new HashSet<>();

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java

@@ -28,6 +28,7 @@ public class AnalyzeResult {
     private boolean wasReanalyzed;
 
     private int analysisVersion;
+    private int analysisNumber;
 
     private ManualRedactions manualRedactions;
 

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Change.java

@@ -13,6 +13,7 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public class Change {
 
+    private int analysisNumber;
     private ChangeType type;
     private OffsetDateTime dateTime;
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java

@@ -18,6 +18,11 @@ public class RedactionLog {
      */
     private long analysisVersion;
 
+    /**
+     * Which analysis created this redactionLog.
+     */
+    private int analysisNumber;
+
     private List<RedactionLogEntry> redactionLogEntry;
     private List<LegalBasis> legalBasis;
 
@@ -26,4 +31,5 @@ public class RedactionLog {
     private long rulesVersion = -1;
     private long legalBasisVersion = -1;
 
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -1,40 +1,30 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.model.*;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
-import org.kie.api.runtime.KieContainer;
+import com.iqser.red.service.redaction.v1.model.*;
-import org.springframework.stereotype.Service;
-import org.springframework.web.bind.annotation.RequestBody;
-
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Slf4j
 @Service
@@ -74,7 +64,7 @@ public class AnalyzeService {
         List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
         sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
 
-        Text text = new Text(pageCount,sectionTexts);
+        Text text = new Text(pageCount, sectionTexts);
 
         // enhance section grid with headline data
         sectionTexts.forEach(sectionText -> classifiedDoc
@@ -97,7 +87,7 @@ public class AnalyzeService {
 
         var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){
+        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
             nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
             nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         }
@@ -116,10 +106,9 @@ public class AnalyzeService {
                 .getDossierTemplateId());
 
         var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary
+        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(),
-                .getVersion()
+                redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
-                .getDossierTemplateVersion(), dictionary.getVersion()
+                rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-                .getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
 
         return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
     }
@@ -149,7 +138,7 @@ public class AnalyzeService {
         }
 
         var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){
+        if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
             nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
             nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         }
@@ -214,7 +203,8 @@ public class AnalyzeService {
 
         excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
 
-        var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
+        var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(),
+                redactionLog, analyzeRequest.getAnalysisNumber());
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange
                 .getRedactionLog());
 
@@ -227,6 +217,7 @@ public class AnalyzeService {
                 .numberOfPages(text.getNumberOfPages())
                 .hasUpdates(redactionLogChange.isHasChanges())
                 .analysisVersion(redactionServiceSettings.getAnalysisVersion())
+                .analysisNumber(analyzeRequest.getAnalysisNumber())
                 .rulesVersion(redactionLog.getRulesVersion())
                 .dictionaryVersion(redactionLog.getDictionaryVersion())
                 .legalBasisVersion(redactionLog.getLegalBasisVersion())
@@ -243,12 +234,12 @@ public class AnalyzeService {
         }
 
         return Stream.concat(manualRedactions.getLegalBasisChanges()
-                .stream()
+                        .stream()
-                .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
+                        .map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
-                .stream()
+                        .stream()
-                .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
+                        .map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
-                .stream()
+                        .stream()
-                .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
+                        .map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
                 .collect(Collectors.toSet());
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
+import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -92,7 +94,7 @@ public class EntityRedactionService {
                         .collect(Collectors.toList());
                 // only approved id removals, that haven't been forced back afterwards
                 var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
-                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED)
+                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
                         .filter(idr -> idr.getRequestDate() != null)
                         .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
                         .map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
@@ -263,7 +265,7 @@ public class EntityRedactionService {
                 .containsKey(sectionNumber)) {
             nerEntities.getResult().get(sectionNumber).forEach(res -> {
                 if (cellstarts == null || cellstarts.isEmpty()) {
-                    nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
+                    nerValuesPerType.computeIfAbsent(RECOMMENDATION_PREFIX + res.getType(), (a) -> new HashSet<>())
                             .add(new String(Base64.decodeBase64(res.getValue().getBytes())));
                 } else {
                     boolean intersectsCellStart = false;
@@ -273,7 +275,7 @@ public class EntityRedactionService {
                         }
                     }
                     if (!intersectsCellStart) {
-                        nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
+                        nerValuesPerType.computeIfAbsent(RECOMMENDATION_PREFIX + res.getType(), (a) -> new HashSet<>())
                                 .add(new String(Base64.decodeBase64(res.getValue().getBytes())));
                     }
                 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -22,7 +22,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 @RequiredArgsConstructor
 public class ManualRedactionSurroundingTextService {
@@ -88,6 +90,11 @@ public class ManualRedactionSurroundingTextService {
 
         Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);
 
+        if(correctEntity == null){
+            log.warn("Could not calculate surrounding text");
+            return Pair.of("","");
+        }
+
         if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
             surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText
                     .getCellStarts());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java

@@ -1,25 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import java.time.OffsetDateTime;
+import com.iqser.red.service.redaction.v1.model.*;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.model.Change;
-import com.iqser.red.service.redaction.v1.model.ChangeType;
-import com.iqser.red.service.redaction.v1.model.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.RedactionLogChanges;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.time.OffsetDateTime;
+import java.util.*;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Service
@@ -29,7 +18,7 @@ public class RedactionChangeLogService {
     private final RedactionStorageService redactionStorageService;
 
 
-    public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) {
+    public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) {
 
         long start = System.currentTimeMillis();
 
@@ -37,7 +26,7 @@ public class RedactionChangeLogService {
 
         if (previousRedactionLog == null) {
             currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
-                entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
             });
             return new RedactionLogChanges(currentRedactionLog, false);
         }
@@ -50,17 +39,16 @@ public class RedactionChangeLogService {
                         .equals(ChangeType.REMOVED))
                 .collect(Collectors.toList());
 
-        Set<RedactionLogEntry> added = new HashSet<>(currentRedactionLog.getRedactionLogEntry().stream()
+        Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream()
                 .filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges()
                         .get(entry.getChanges().size() - 1)
                         .getType()
-                        .equals(ChangeType.REMOVED))
+                        .equals(ChangeType.REMOVED)).collect(Collectors.toSet());
-                .collect(Collectors.toList()));
 
-        added.removeAll(notRemovedPreviousEntries);
+        notRemovedPreviousEntries.forEach(added::remove);
 
         Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
-        removed.removeAll(currentRedactionLog.getRedactionLogEntry());
+        currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
 
         Map<String, RedactionLogEntry> addedIds = new HashMap<>();
         added.forEach(entry -> {
@@ -78,16 +66,16 @@ public class RedactionChangeLogService {
         newRedactionLogEntries.forEach(entry -> {
             if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
                 List<Change> changes = entry.getChanges();
-                changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now()));
+                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
                 var newEntry = addedIds.get(entry.getId());
                 newEntry.setChanges(changes);
                 addedIds.put(entry.getId(), newEntry);
                 toRemove.add(entry);
             } else if (removedIds.contains(entry.getId())) {
-                entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now()));
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
             } else if (addedIds.containsKey(entry.getId())) {
                 List<Change> changes = entry.getChanges();
-                changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
+                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
                 var newEntry = addedIds.get(entry.getId());
                 newEntry.setChanges(changes);
                 addedIds.put(entry.getId(), newEntry);
@@ -98,8 +86,8 @@ public class RedactionChangeLogService {
         newRedactionLogEntries.removeAll(toRemove);
 
         addedIds.forEach((k, v) -> {
-            if(v.getChanges().isEmpty()) {
+            if (v.getChanges().isEmpty()) {
-                v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
+                v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now()));
             }
             newRedactionLogEntries.add(v);
         });

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -128,7 +128,12 @@ public class EntitySearchUtils {
             for (Entity inner : entities) {
                 if (inner.getWord().length() < word.getWord()
                         .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) {
-                    wordsToRemove.add(inner);
+                    // FIXME this is workaround for RED-3327 and should be removed in the future.
+                    if(word.getType().contains("recommendation_") && !inner.getType().contains("recommendation_")) {
+                        wordsToRemove.add(word);
+                    } else {
+                        wordsToRemove.add(inner);
+                    }
                 }
             }
         }