RED-8702: Explore document databases to store entityLog

* wip - rewrite entity log creation and update with mongodb

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -1,5 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import static com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService.getRelevantManuallyModifiedAnnotationIds;
+
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -20,6 +23,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
@@ -80,7 +84,7 @@ public class AnalyzeService {
     public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
 
         long startTime = System.currentTimeMillis();
-        EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        EntityLog entityLogWithoutEntries = redactionStorageService.getEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
@@ -90,25 +94,37 @@ public class AnalyzeService {
         log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         // not yet ready for reanalysis
-        if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
+        if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) {
             return analyze(analyzeRequest);
         }
 
         DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
-                                                                                            new DictionaryVersion(previousEntityLog.getDictionaryVersion(),
+                                                                                            new DictionaryVersion(entityLogWithoutEntries.getDictionaryVersion(),
-                                                                                                                  previousEntityLog.getDossierDictionaryVersion()),
+                                                                                                                  entityLogWithoutEntries.getDossierDictionaryVersion()),
                                                                                             analyzeRequest.getDossierId());
 
-        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions);
+        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+
+        List<EntityLogEntry> relevantEntityLogEntries = redactionStorageService.findEntriesWithContainingNodeIds(analyzeRequest.getDossierId(),
+                                                                                                                 analyzeRequest.getFileId(),
+                                                                                                                 relevantManuallyModifiedAnnotationIds);
+        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest,
+                                                                        document,
+                                                                        dictionaryIncrement,
+                                                                        importedRedactions,
+                                                                        relevantEntityLogEntries,
+                                                                        relevantManuallyModifiedAnnotationIds);
+
         List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
         log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         if (sectionsToReAnalyse.isEmpty()) {
 
-            EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
+            // todo: 8702 - this feels  wrong
+            EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
                                                                                                        dictionaryIncrement.getDictionaryVersion(),
                                                                                                        analyzeRequest,
-                                                                                                       false);
+                                                                                                       new ArrayList<>(), new ArrayList<>());
 
             return finalizeAnalysis(analyzeRequest,
                                     startTime,
@@ -154,7 +170,7 @@ public class AnalyzeService {
         EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
                                                                                             document,
                                                                                             notFoundManualOrImportedEntries,
-                                                                                            previousEntityLog,
+                                                                                            entityLogWithoutEntries,
                                                                                             sectionsToReanalyseIds,
                                                                                             dictionary.getVersion());
 
@@ -228,7 +244,7 @@ public class AnalyzeService {
         return finalizeAnalysis(analyzeRequest,
                                 startTime,
                                 kieWrapperComponentRules,
-                                new EntityLogChanges(entityLog, false),
+                                EntityLogChanges.builder().entityLog(entityLog).build(),
                                 document,
                                 document.getNumberOfPages(),
                                 dictionary.getVersion(),
@@ -248,7 +264,7 @@ public class AnalyzeService {
                                            Set<FileAttribute> addedFileAttributes) {
 
         EntityLog entityLog = entityLogChanges.getEntityLog();
-        //redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+
         redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog());
 
         log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -317,12 +333,18 @@ public class AnalyzeService {
 
 
     private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest,
-                                                   EntityLog entityLog,
                                                    Document document,
                                                    DictionaryIncrement dictionaryIncrement,
-                                                   ImportedRedactions importedRedactions) {
+                                                   ImportedRedactions importedRedactions,
+                                                   List<EntityLogEntry> entityLogEntries,
+                                                   Set<String> relevantManuallyModifiedAnnotationIds) {
 
-        return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions);
+        return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement,
+                                                            document,
+                                                            analyzeRequest,
+                                                            importedRedactions,
+                                                            entityLogEntries,
+                                                            relevantManuallyModifiedAnnotationIds);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -95,7 +95,7 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, boolean hasChanges) {
+    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List<EntityLogEntry> newEntries, List<EntityLogEntry> updatedEntries) {
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
@@ -104,7 +104,7 @@ public class EntityLogCreatorService {
         entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
         entityLog.setAnalysisNumber(analyzeRequest.getAnalysisNumber());
 
-        return new EntityLogChanges(entityLog, hasChanges);
+        return EntityLogChanges.builder().entityLog(entityLog).newEntityLogEntries(newEntries).updatedEntityLogEntries(updatedEntries).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -11,7 +11,6 @@ import java.util.stream.Stream;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
@@ -44,23 +43,17 @@ public class SectionFinderService {
 
     @Timed("redactmanager_findSectionsToReanalyse")
     public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement,
-                                                EntityLog entityLog,
                                                 Document document,
                                                 AnalyzeRequest analyzeRequest,
-                                                ImportedRedactions importedRedactions) {
+                                                ImportedRedactions importedRedactions,
+                                                List<EntityLogEntry> entityLogEntries,
+                                                Set<String> relevantManuallyModifiedAnnotationIds) {
 
         long start = System.currentTimeMillis();
-        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
         Set<Integer> sectionsToReanalyse = new HashSet<>();
-        for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
+
-            if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
+        entityLogEntries.forEach(entityLogEntry -> sectionsToReanalyse.add(entityLogEntry.getContainingNodeId()
-                if (entry.getContainingNodeId().isEmpty()) {
+                                                                                   .get(0)));
-                    continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
-                }
-                sectionsToReanalyse.add(entry.getContainingNodeId()
-                                                .get(0));
-            }
-        }
 
         var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
                                                                           .stream()
@@ -133,7 +126,7 @@ public class SectionFinderService {
     }
 
 
-    private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
+    public static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
             return new HashSet<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.storage;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
+import java.util.Collection;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -10,6 +11,7 @@ import org.springframework.cache.annotation.Cacheable;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@@ -142,8 +144,8 @@ public class RedactionStorageService {
     public EntityLog getEntityLog(String dossierId, String fileId) {
 
         try {
-            //EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
+            EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId)
-            EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId).orElseThrow(() -> new StorageObjectDoesNotExist(""));
+                    .orElseThrow(() -> new StorageObjectDoesNotExist(""));
             entityLog.setEntityLogEntry(entityLog.getEntityLogEntry()
                                                 .stream()
                                                 .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
@@ -157,6 +159,26 @@ public class RedactionStorageService {
     }
 
 
+    @Timed("redactmanager_getRedactionLog")
+    public EntityLog getEntityLogWithoutEntries(String dossierId, String fileId) {
+
+        try {
+            return entityLogMongoService.findEntityLogWithoutEntries(dossierId, fileId)
+                    .orElseThrow(() -> new StorageObjectDoesNotExist(""));
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("EntityLog not available.");
+            return null;
+        }
+
+    }
+
+
+    public List<EntityLogEntry> findEntriesWithContainingNodeIds(String dossierId, String fileId, Collection<String> entryIds) {
+
+        return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds);
+    }
+
+
     // !Warning! before activating redis cache you need to set
     // -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M
     // Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb.