From a4165608d428ecc147a60a3f93640cb92faca152 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Mon, 8 Jul 2024 13:04:54 +0200 Subject: [PATCH] RED-9123: Improve performance of re-analysis (Spike) * refactoring of Update previous entity log logic --- .../service/EntityChangeLogService.java | 2 + .../service/EntityLogCreatorService.java | 82 +++++++++++++------ .../storage/RedactionStorageService.java | 1 + .../v1/server/AnalysisEnd2EndTest.java | 61 +++++++++++++- 4 files changed, 117 insertions(+), 29 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index 156856b6..e687ccc3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -22,6 +22,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker; import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; @@ -34,6 +35,7 @@ import lombok.extern.slf4j.Slf4j; public class EntityChangeLogService { @Timed("redactmanager_computeChanges") + @Observed(name = "EntityChangeLogService", contextualName = "compute-changes") public EntryChanges computeChanges(List previousEntityLogEntries, List newEntityLogEntries, int analysisNumber) { var now = OffsetDateTime.now(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index a4b900c6..230591e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -2,12 +2,16 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.springframework.stereotype.Service; @@ -44,6 +48,8 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils; import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationRegistry; import io.micrometer.observation.annotation.Observed; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -62,6 +68,7 @@ public class EntityLogCreatorService { EntityChangeLogService entityChangeLogService; RedactionStorageService redactionStorageService; EntityLogMongoService entityLogMongoService; + ObservationRegistry observationRegistry; private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) { @@ -72,7 +79,6 @@ public class EntityLogCreatorService { } - @Timed("redactmanager_createInitialEntityLog") @Observed(name = "EntityLogCreatorService", contextualName = "create-initial-entity-log") public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest, @@ -114,7 +120,6 @@ public class EntityLogCreatorService { } - @Timed("redactmanager_updateVersionsAndReturnChanges") @Observed(name = "EntityLogCreatorService", contextualName = "update-versions-and-return-changes") public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, @@ -143,16 +148,23 @@ public class EntityLogCreatorService { Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { - List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream() - .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() - .get(0))) - .collect(Collectors.toList()); + AtomicReference> newEntityLogEntriesReference = new AtomicReference<>(); + + Observation.createNotStarted("EntityLogCreatorService", observationRegistry) + .contextualName("create-new-entity-log-entries") + .observe(() -> newEntityLogEntriesReference.set(createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream() + .filter(entry -> entry.getContainingNodeId().isEmpty() + || sectionsToReanalyseIds.contains(entry.getContainingNodeId() + .get(0))) + .collect(Collectors.toList()))); List previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), sectionsToReanalyseIds); - EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); + EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, + newEntityLogEntriesReference.get(), + analyzeRequest.getAnalysisNumber()); return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated()); } @@ -164,26 +176,47 @@ public class EntityLogCreatorService { List entries = new ArrayList<>(); - document.getEntities() + List textEntities = document.getEntities() .stream() .filter(entity -> !entity.getValue().isEmpty()) .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval) .filter(entity -> !entity.removed()) - .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + .toList(); - document.streamAllImages() + List images = document.streamAllImages() .filter(entity -> !entity.removed()) - .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + .toList(); - notFoundPrecursorEntries.stream() + List notFoundPrecursorEntities = notFoundPrecursorEntries.stream() .filter(entity -> !entity.removed()) - .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + .toList(); + + List allIds = new ArrayList<>(); + allIds.addAll(textEntities.stream() + .flatMap(entity -> entity.getPositionsOnPagePerPage() + .stream() + .map(PositionOnPage::getId)) + .toList()); + allIds.addAll(images.stream() + .map(Image::getId) + .toList()); + allIds.addAll(notFoundPrecursorEntities.stream() + .map(PrecursorEntity::getId) + .toList()); + + Map> manualChangesMap = getManualChangesByEntityLogIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), allIds); + + textEntities.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, manualChangesMap))); + + images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>())))); + + textEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>())))); return entries; } - private List toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) { + private List toEntityLogEntries(TextEntity textEntity, int analysisNumber, Map> existingManualChangesMap) { List entityLogEntries = new ArrayList<>(); @@ -195,7 +228,7 @@ public class EntityLogCreatorService { .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) .toList(); - EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId); + EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, existingManualChangesMap.getOrDefault(positionOnPage.getId(), new ArrayList<>())); // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages entityLogEntry.setId(positionOnPage.getId()); @@ -207,12 +240,11 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) { + private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, List existingManualChanges) { String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); - List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId()); List allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber); return EntityLogEntry.builder() @@ -240,12 +272,11 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) { + private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, List existingManualChanges) { String type = precursorEntity.getManualOverwrite().getType() .orElse(precursorEntity.getType()); - List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId()); List allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); return EntityLogEntry.builder() @@ -283,7 +314,7 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) { + private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, List existingManualChanges) { Set referenceIds = new HashSet<>(); entity.references() @@ -294,7 +325,6 @@ public class EntityLogCreatorService { EntryType entryType = buildEntryType(entity); - List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id); List allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); return EntityLogEntry.builder() @@ -431,16 +461,16 @@ public class EntityLogCreatorService { } - public List getManualChangesByEntityLogId(String dossierId, String fileId, String id) { + public Map> getManualChangesByEntityLogIds(String dossierId, String fileId, List ids) { - List manualChanges = new ArrayList<>(); - List entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id)); + Map> manualChangesMap = new HashMap<>(); + List entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, ids); for (EntityLogEntry entry : entityLogEntries) { - manualChanges.addAll(entry.getManualChanges()); + manualChangesMap.put(entry.getId(), entry.getManualChanges()); } - return manualChanges; + return manualChangesMap; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 69cfd5d3..5da87610 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -36,6 +36,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do import com.knecon.fforesight.tenantcommons.TenantContext; import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index 170e417f..f45527ab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -9,9 +9,15 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; +import java.time.OffsetDateTime; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -40,14 +46,19 @@ import org.springframework.data.redis.listener.RedisMessageListenerContainer; import org.springframework.test.context.junit.jupiter.SpringExtension; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.common.collect.Sets; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; @@ -85,8 +96,7 @@ import lombok.extern.slf4j.Slf4j; * This way you can recreate what is happening on the stack almost exactly. */ public class AnalysisEnd2EndTest { - Path dossierTemplateToUse = Path.of( - "/Users/maverickstuder/Documents/syngenta/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here + Path dossierTemplateToUse = Path.of("/Users/maverickstuder/Documents/syngenta/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here ObjectMapper mapper = ObjectMapperFactory.create(); final String TENANT_ID = "tenant"; @@ -131,6 +141,9 @@ import lombok.extern.slf4j.Slf4j; @MockBean private RedisMessageListenerContainer redisPubsubContainer; + @Autowired + RedactionStorageService redactionStorageService; + @Test @SneakyThrows @@ -155,7 +168,49 @@ import lombok.extern.slf4j.Slf4j; var times = new LinkedList(); for (int j = 1; j <= 1; j++) { var start = System.currentTimeMillis(); - analyzeService.analyze(analyzeRequest); + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + +// analyzeService.analyze(analyzeRequest); +// EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); +// +// // Serialize +// String jsonString = objectMapper.writeValueAsString(entityLog); +// // Serialize entityLog to a file +// try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); +// ObjectOutputStream out = new ObjectOutputStream(fileOut)) { +// out.writeObject(jsonString); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// + + + //Deserialize entityLog from a file + String entityLogString = null; + try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) { + entityLogString = (String) in.readObject(); + } catch (IOException | ClassNotFoundException e) { + e.printStackTrace(); + } + var entityLog = objectMapper.readValue(entityLogString, EntityLog.class); + redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId("yourAnnotationId"); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setType("CBI_author"); + manualRedactionEntry.setValue("7232"); + manualRedactionEntry.setReason( + "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)"); + manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build())); + + analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build()); + + analyzeService.reanalyze(analyzeRequest); times.add(System.currentTimeMillis() - start); } System.out.println("times in ms for each analyze run: " + times);