RED-9123: Improve performance of re-analysis (Spike)

* further async processing for finalize analysis as well as addDictionaryEntities

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 
 import org.springframework.stereotype.Service;
 
@@ -56,23 +57,27 @@ public class AnalysisFinalizationService {
 
         EntityLog entityLog = entityLogChanges.getEntityLog();
 
-        // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
+        CompletableFuture.runAsync(() -> {
-        // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
-        if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
-            redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
 
-        } else {
+            // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
-            redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+            // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
+            if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
+                redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
 
-            if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
+            } else {
-                redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
+                redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+                if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
+                    redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
+                }
+                if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
+                    redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
+                }
             }
-            if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
-                redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
-            }
-        }
 
-        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+            log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        });
 
         computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -39,14 +39,27 @@ public class DictionarySearchService {
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+        dictionary.getDictionaryModels()
-            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
+                .stream()
-            bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
+                .parallel()
-            bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
+                .forEach(model -> {
-            if (model.isDossierDictionary()) {
+                    synchronized (node) {
-                bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
+                        bySearchImplementationAsDictionary(model.getEntriesSearch(),
-            }
+                                                           model.getType(),
-        }
+                                                           model.isHint() ? EntityType.HINT : EntityType.ENTITY,
+                                                           node,
+                                                           model.isDossierDictionary());
+                        bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
+                        bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
+                                                           model.getType(),
+                                                           EntityType.FALSE_RECOMMENDATION,
+                                                           node,
+                                                           model.isDossierDictionary());
+                        if (model.isDossierDictionary()) {
+                            bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
+                        }
+                    }
+                });
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

@@ -13,7 +13,6 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
-import java.io.Serializable;
 import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -33,7 +32,6 @@ import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
@@ -45,6 +43,7 @@ import org.springframework.core.io.ClassPathResource;
 import org.springframework.data.redis.listener.RedisMessageListenerContainer;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.google.common.collect.Sets;
@@ -58,7 +57,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@@ -171,44 +169,12 @@ import lombok.extern.slf4j.Slf4j;
                 ObjectMapper objectMapper = new ObjectMapper();
                 objectMapper.registerModule(new JavaTimeModule());
 
-//                analyzeService.analyze(analyzeRequest);
+                //analyzeService.analyze(analyzeRequest);
-//                EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                //storeEntityLogAsTempFile(analyzeRequest, objectMapper);
-//
-//                // Serialize
-//                String jsonString = objectMapper.writeValueAsString(entityLog);
-//                // Serialize entityLog to a file
-//                try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser");
-//                     ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
-//                    out.writeObject(jsonString);
-//                } catch (IOException e) {
-//                    e.printStackTrace();
-//                }
-//
 
+                restoreEntityLogFromTempFile(objectMapper, analyzeRequest);
 
-                //Deserialize entityLog from a file
+                addManualRedactionEntryToFile(analyzeRequest);
-                String entityLogString = null;
-                try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
-                    entityLogString = (String) in.readObject();
-                } catch (IOException | ClassNotFoundException e) {
-                    e.printStackTrace();
-                }
-                var entityLog = objectMapper.readValue(entityLogString, EntityLog.class);
-                redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
-
-                ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
-                manualRedactionEntry.setAnnotationId("yourAnnotationId");
-                manualRedactionEntry.setFileId("fileId");
-                manualRedactionEntry.setType("CBI_author");
-                manualRedactionEntry.setValue("7232");
-                manualRedactionEntry.setReason(
-                        "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)");
-                manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
-                manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
-                manualRedactionEntry.setRequestDate(OffsetDateTime.now());
-                manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build()));
-
-                analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
 
                 analyzeService.reanalyze(analyzeRequest);
                 times.add(System.currentTimeMillis() - start);
@@ -218,6 +184,53 @@ import lombok.extern.slf4j.Slf4j;
     }
 
 
+    private static void addManualRedactionEntryToFile(AnalyzeRequest analyzeRequest) {
+
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId("yourAnnotationId");
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setType("CBI_author");
+        manualRedactionEntry.setValue("7232");
+        manualRedactionEntry.setReason(
+                "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)");
+        manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+        manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
+        manualRedactionEntry.setRequestDate(OffsetDateTime.now());
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build()));
+
+        analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
+    }
+
+
+    private void restoreEntityLogFromTempFile(ObjectMapper objectMapper, AnalyzeRequest analyzeRequest) throws JsonProcessingException {
+        //Deserialize entityLog from a file
+        String entityLogString = null;
+        try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
+            entityLogString = (String) in.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+        var entityLog = objectMapper.readValue(entityLogString, EntityLog.class);
+        redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+    }
+
+
+    private void storeEntityLogAsTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException {
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+
+        // Serialize
+        String jsonString = objectMapper.writeValueAsString(entityLog);
+        // Serialize entityLog to a file
+        try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser");
+             ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
+            out.writeObject(jsonString);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+
     @BeforeEach
     public void setup() {