RED-9123: Improve performance of re-analysis (Spike)

* refactoring of Update previous entity log logic

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker;
 
 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -34,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityChangeLogService {
 
     @Timed("redactmanager_computeChanges")
+    @Observed(name = "EntityChangeLogService", contextualName = "compute-changes")
     public EntryChanges computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
 
         var now = OffsetDateTime.now();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -2,12 +2,16 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 
@@ -44,6 +48,8 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService
 import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils;
 
 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -62,6 +68,7 @@ public class EntityLogCreatorService {
     EntityChangeLogService entityChangeLogService;
     RedactionStorageService redactionStorageService;
     EntityLogMongoService entityLogMongoService;
+    ObservationRegistry observationRegistry;
 
 
     private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
@@ -72,7 +79,6 @@ public class EntityLogCreatorService {
     }
 
 
-
     @Timed("redactmanager_createInitialEntityLog")
     @Observed(name = "EntityLogCreatorService", contextualName = "create-initial-entity-log")
     public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
@@ -114,7 +120,6 @@ public class EntityLogCreatorService {
     }
 
 
-
     @Timed("redactmanager_updateVersionsAndReturnChanges")
     @Observed(name = "EntityLogCreatorService", contextualName = "update-versions-and-return-changes")
     public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
@@ -143,16 +148,23 @@ public class EntityLogCreatorService {
                                                     Set<Integer> sectionsToReanalyseIds,
                                                     DictionaryVersion dictionaryVersion) {
 
-        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream()
+        AtomicReference<List<EntityLogEntry>> newEntityLogEntriesReference = new AtomicReference<>();
-                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
+
+        Observation.createNotStarted("EntityLogCreatorService", observationRegistry)
+                .contextualName("create-new-entity-log-entries")
+                .observe(() -> newEntityLogEntriesReference.set(createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream()
+                                                                        .filter(entry -> entry.getContainingNodeId().isEmpty()
+                                                                                         || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
                                                                                                                                     .get(0)))
-                .collect(Collectors.toList());
+                                                                        .collect(Collectors.toList())));
 
         List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(),
                                                                                                                                           analyzeRequest.getFileId(),
                                                                                                                                           sectionsToReanalyseIds);
 
-        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
+                                                                          newEntityLogEntriesReference.get(),
+                                                                          analyzeRequest.getAnalysisNumber());
 
         return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
     }
@@ -164,26 +176,47 @@ public class EntityLogCreatorService {
 
         List<EntityLogEntry> entries = new ArrayList<>();
 
-        document.getEntities()
+        List<TextEntity> textEntities = document.getEntities()
                 .stream()
                 .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
+                .toList();
 
-        document.streamAllImages()
+        List<Image> images = document.streamAllImages()
                 .filter(entity -> !entity.removed())
-                .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
+                .toList();
 
-        notFoundPrecursorEntries.stream()
+        List<PrecursorEntity> notFoundPrecursorEntities = notFoundPrecursorEntries.stream()
                 .filter(entity -> !entity.removed())
-                .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
+                .toList();
+
+        List<String> allIds = new ArrayList<>();
+        allIds.addAll(textEntities.stream()
+                              .flatMap(entity -> entity.getPositionsOnPagePerPage()
+                                      .stream()
+                                      .map(PositionOnPage::getId))
+                              .toList());
+        allIds.addAll(images.stream()
+                              .map(Image::getId)
+                              .toList());
+        allIds.addAll(notFoundPrecursorEntities.stream()
+                              .map(PrecursorEntity::getId)
+                              .toList());
+
+        Map<String, List<ManualChange>> manualChangesMap = getManualChangesByEntityLogIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), allIds);
+
+        textEntities.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, manualChangesMap)));
+
+        images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>()))));
+
+        textEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>()))));
 
         return entries;
     }
 
 
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) {
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, Map<String, List<ManualChange>> existingManualChangesMap) {
 
         List<EntityLogEntry> entityLogEntries = new ArrayList<>();
 
@@ -195,7 +228,7 @@ public class EntityLogCreatorService {
                     .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId);
+            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, existingManualChangesMap.getOrDefault(positionOnPage.getId(), new ArrayList<>()));
 
             // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
             entityLogEntry.setId(positionOnPage.getId());
@@ -207,12 +240,11 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) {
+    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, List<ManualChange> existingManualChanges) {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
 
-        List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId());
         List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
 
         return EntityLogEntry.builder()
@@ -240,12 +272,11 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) {
+    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, List<ManualChange> existingManualChanges) {
 
         String type = precursorEntity.getManualOverwrite().getType()
                 .orElse(precursorEntity.getType());
 
-        List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId());
         List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
 
         return EntityLogEntry.builder()
@@ -283,7 +314,7 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, List<ManualChange> existingManualChanges) {
 
         Set<String> referenceIds = new HashSet<>();
         entity.references()
@@ -294,7 +325,6 @@ public class EntityLogCreatorService {
 
         EntryType entryType = buildEntryType(entity);
 
-        List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id);
         List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
 
         return EntityLogEntry.builder()
@@ -431,16 +461,16 @@ public class EntityLogCreatorService {
     }
 
 
-    public List<ManualChange> getManualChangesByEntityLogId(String dossierId, String fileId, String id) {
+    public Map<String, List<ManualChange>> getManualChangesByEntityLogIds(String dossierId, String fileId, List<String> ids) {
 
-        List<ManualChange> manualChanges = new ArrayList<>();
+        Map<String, List<ManualChange>> manualChangesMap = new HashMap<>();
-        List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id));
+        List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, ids);
 
         for (EntityLogEntry entry : entityLogEntries) {
-            manualChanges.addAll(entry.getManualChanges());
+            manualChangesMap.put(entry.getId(), entry.getManualChanges());
         }
 
-        return manualChanges;
+        return manualChangesMap;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -36,6 +36,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

@@ -9,9 +9,15 @@ import static org.mockito.Mockito.when;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
 import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.OffsetDateTime;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -40,14 +46,19 @@ import org.springframework.data.redis.listener.RedisMessageListenerContainer;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.google.common.collect.Sets;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@@ -85,8 +96,7 @@ import lombok.extern.slf4j.Slf4j;
  * This way you can recreate what is happening on the stack almost exactly.
  */ public class AnalysisEnd2EndTest {
 
-    Path dossierTemplateToUse = Path.of(
+    Path dossierTemplateToUse = Path.of("/Users/maverickstuder/Documents/syngenta/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here
-            "/Users/maverickstuder/Documents/syngenta/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here
     ObjectMapper mapper = ObjectMapperFactory.create();
     final String TENANT_ID = "tenant";
 
@@ -131,6 +141,9 @@ import lombok.extern.slf4j.Slf4j;
     @MockBean
     private RedisMessageListenerContainer redisPubsubContainer;
 
+    @Autowired
+    RedactionStorageService redactionStorageService;
+
 
     @Test
     @SneakyThrows
@@ -155,7 +168,49 @@ import lombok.extern.slf4j.Slf4j;
             var times = new LinkedList<Long>();
             for (int j = 1; j <= 1; j++) {
                 var start = System.currentTimeMillis();
-                analyzeService.analyze(analyzeRequest);
+                ObjectMapper objectMapper = new ObjectMapper();
+                objectMapper.registerModule(new JavaTimeModule());
+
+//                analyzeService.analyze(analyzeRequest);
+//                EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+//
+//                // Serialize
+//                String jsonString = objectMapper.writeValueAsString(entityLog);
+//                // Serialize entityLog to a file
+//                try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser");
+//                     ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
+//                    out.writeObject(jsonString);
+//                } catch (IOException e) {
+//                    e.printStackTrace();
+//                }
+//
+
+
+                //Deserialize entityLog from a file
+                String entityLogString = null;
+                try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
+                    entityLogString = (String) in.readObject();
+                } catch (IOException | ClassNotFoundException e) {
+                    e.printStackTrace();
+                }
+                var entityLog = objectMapper.readValue(entityLogString, EntityLog.class);
+                redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+                ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+                manualRedactionEntry.setAnnotationId("yourAnnotationId");
+                manualRedactionEntry.setFileId("fileId");
+                manualRedactionEntry.setType("CBI_author");
+                manualRedactionEntry.setValue("7232");
+                manualRedactionEntry.setReason(
+                        "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)");
+                manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+                manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
+                manualRedactionEntry.setRequestDate(OffsetDateTime.now());
+                manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build()));
+
+                analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
+
+                analyzeService.reanalyze(analyzeRequest);
                 times.add(System.currentTimeMillis() - start);
             }
             System.out.println("times in ms for each analyze run: " + times);