RED-10425 - Annotation added twice when bulk-force while auto-analysis is disabled

- use a custom query to get the dictionary entity log entries with the found value and positions list needed instead of the entire entity log
- add index for entityLogId and value

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/SearchTermOccurrencesResponseReceiver.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.queue;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -13,7 +12,6 @@ import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
 import org.springframework.security.core.Authentication;
-import org.springframework.security.core.context.SecurityContext;
 import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.security.core.context.SecurityContextImpl;
 import org.springframework.stereotype.Service;
@@ -22,7 +20,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
-import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
 import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.AuditPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
@@ -30,8 +27,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.websock
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FoundTerm;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualAnnotationResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@@ -39,6 +35,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
 import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
 
 import lombok.AccessLevel;
@@ -61,7 +58,7 @@ public class SearchTermOccurrencesResponseReceiver {
     AuditPersistenceService auditPersistenceService;
     FileStatusPersistenceService fileStatusPersistenceService;
     WebsocketService webSocketService;
-    EntityLogService entityLogService;
+    EntityLogMongoService entityLogMongoService;
 
 
     @RabbitHandler
@@ -91,20 +88,26 @@ public class SearchTermOccurrencesResponseReceiver {
 
         var dossier = dossierManagementService.getDossierById(response.getDossierId(), false, false);
 
-        var entityLog = entityLogService.getEntityLog(response.getDossierId(), response.getFileId(), new ArrayList<>(), false);
+        var foundValue = response.getFoundTerms()
+                .get(0).value();
+        List<List<Position>> positionsList = response.getFoundTerms()
+                .stream()
+                .map(FoundTerm::positions)
+                .collect(Collectors.toList());
+        var entityLogEntries = entityLogMongoService.findDictionaryEntityLogEntriesByValueAndPositions(response.getDossierId(), response.getFileId(), foundValue, positionsList);
 
         Set<AddRedactionRequestModel> addRedactionRequests = response.getFoundTerms()
                 .stream()
                 .map(term -> AddRedactionRequestModel.builder()
-                            .type(response.getType())
-                            .value(term.value())
-                            .reason(response.getReason())
-                            .legalBasis(response.getLegalBasis())
-                            .positions(convertPositions(term.positions()))
-                            .section(response.getSection())
-                            .comment(response.getComment() == null ? null : new AddCommentRequestModel(response.getComment()))
-                            .sourceId(getAnnotationId(entityLog, term))
-                            .build())
+                        .type(response.getType())
+                        .value(term.value())
+                        .reason(response.getReason())
+                        .legalBasis(response.getLegalBasis())
+                        .positions(convertPositions(term.positions()))
+                        .section(response.getSection())
+                        .comment(response.getComment() == null ? null : new AddCommentRequestModel(response.getComment()))
+                        .sourceId(getAnnotationId(entityLogEntries, term))
+                        .build())
                 .collect(Collectors.toSet());
 
         log.info("Received manual redaction requests for file {} in dossier {} after term search", response.getFileId(), dossier.getId());
@@ -134,13 +137,10 @@ public class SearchTermOccurrencesResponseReceiver {
     }
 
 
-    private String getAnnotationId(EntityLog entityLog, FoundTerm term) {
+    private String getAnnotationId(Set<EntityLogEntry> entityLogEntries, FoundTerm term) {
 
-        var optionalEntry = entityLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> entry.getValue().equals(term.value())
-                                 && entry.getPositions().equals(term.positions())
-                                 && (entry.isDictionaryEntry() || entry.isDossierDictionaryEntry()))
+        var optionalEntry = entityLogEntries.stream()
+                .filter(entry -> entry.getPositions().equals(term.positions()))
                 .findFirst();
         if (optionalEntry.isPresent()) {
             return optionalEntry.get().getId();

persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml

@@ -4,9 +4,10 @@
         xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd">
     <include file="/mongo/changelog/tenant/1-initial-database.changelog.xml"/>
     <include file="/mongo/changelog/tenant/2-create-indices-for-entries.xml"/>
-    <include file="/mongo/changelog/tenant/3-add-page-paragraph-idx.xml"/>
-    <!--<include file="/mongo/changelog/tenant/4-create-component-entities.xml"/>-->
-    <include file="/mongo/changelog/tenant/5-add-duplicate-text-ranges.xml"/>
-    <include file="/mongo/changelog/tenant/6-rename-component-collections.xml"/>
+<!--    <include file="/mongo/changelog/tenant/3-add-page-paragraph-idx.xml"/>-->
+<!--    <include file="/mongo/changelog/tenant/4-create-component-entities.xml"/>-->
+<!--    <include file="/mongo/changelog/tenant/5-add-duplicate-text-ranges.xml"/>-->
+<!--    <include file="/mongo/changelog/tenant/6-rename-component-collections.xml"/>-->
+    <include file="/mongo/changelog/tenant/7-add-entity-log-value-index.xml"/>
     <!-- THIS FILE IS NOT RUN IN THE CURRENT CONFIGURATION, PLEASE ADD CHANGES TO redaction-service -->
 </databaseChangeLog>

persistence-service-v1/persistence-service-processor-v1/src/main/resources/mongo/changelog/tenant/7-add-entity-log-value-index.xml

@@ -0,0 +1,23 @@
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <changeSet id="createIndexForEntityLogIdAndValue" author="corina">
+
+        <ext:createIndex collectionName="entity-log-entries">
+            <ext:keys>
+                {
+                "entityLogId": 1,
+                "value": 1,
+                }
+            </ext:keys>
+            <ext:options>
+                {name: "entityLogId_value_index"}
+            </ext:options>
+        </ext:createIndex>
+
+    </changeSet>
+</databaseChangeLog>

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java

@@ -4247,7 +4247,6 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
                                      .dossierDictionaryEntry(true)
                                      .positions(position4)
                                      .build());
-//        }
         var entityLog = new EntityLog(1, 1, entityLogEntries, null, 0, 0, 0, 0);
         fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
 

persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/repository/EntityLogEntryDocumentRepository.java

@@ -8,6 +8,7 @@ import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.stereotype.Repository;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.projections.EntryWithManualChangesProjection;
 
@@ -91,4 +92,13 @@ public interface EntityLogEntryDocumentRepository extends MongoRepository<Entity
         """, fields = "{ 'entryId': 1, 'entityLogId': 1, 'state': 1, 'entryType': 1 , 'manualChanges': 1}")
     List<EntryWithManualChangesProjection> findAppliedEntitiesWhereLegalBasisEmpty();
 
+    @Query("{ 'entityLogId': ?0," +
+           "  'value': ?1, " +
+           "  '$or': [ { 'dictionaryEntry': true }, { 'dossierDictionaryEntry': true } ], " +
+           "  'positions': { $in: ?2 } " +
+           "}")
+    List<EntityLogEntryDocument> findDictionaryEntityLogEntriesByValueAndPositions(String entityLogId,
+                                                                                   String value,
+                                                                                   List<List<Position>> positionsList);
+
 }

persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/EntityLogMongoService.java

@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.exception.DocumentNotFoundException;
@@ -407,4 +408,17 @@ public class EntityLogMongoService {
                 .collect(Collectors.toSet());
     }
 
+    public Set<EntityLogEntry> findDictionaryEntityLogEntriesByValueAndPositions(String dossierId,
+                                                                                 String fileId,
+                                                                                 String value,
+                                                                                 List<List<Position>> positionsList) {
+
+        return new HashSet<>(entityLogEntryDocumentRepository.findDictionaryEntityLogEntriesByValueAndPositions(mapper.getLogId(dossierId, fileId),
+                                                                                                                value,
+                                                                                                                positionsList)
+                                            .stream()
+                                            .map(mapper::fromLogEntryDocument)
+                                            .toList());
+    }
+
 }