From e6a792377b6dcd1a562a87ab82548e115a828dc5 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Mon, 8 Jul 2024 13:58:19 +0200 Subject: [PATCH] RED-9123: Improve performance of re-analysis (Spike) * further async processing for finalize analysis as well as addDictionaryEntities --- .../service/AnalysisFinalizationService.java | 31 ++++--- .../service/DictionarySearchService.java | 29 ++++-- .../v1/server/AnalysisEnd2EndTest.java | 91 +++++++++++-------- 3 files changed, 91 insertions(+), 60 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java index af0f2e8c..a7cabe8e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.List; import java.util.Set; +import java.util.concurrent.CompletableFuture; import org.springframework.stereotype.Service; @@ -56,23 +57,27 @@ public class AnalysisFinalizationService { EntityLog entityLog = entityLogChanges.getEntityLog(); - // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used - // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed - if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) { - redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + CompletableFuture.runAsync(() -> { - } else { - redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used + // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed + if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) { + redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); - if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) { - redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries()); + } else { + redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + + if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) { + redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries()); + } + if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) { + redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries()); + } } - if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) { - redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries()); - } - } - log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + + }); computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 06d8237a..4e2b9d80 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -39,14 +39,27 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { - for (DictionaryModel model : dictionary.getDictionaryModels()) { - bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); - bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); - if (model.isDossierDictionary()) { - bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary()); - } - } + dictionary.getDictionaryModels() + .stream() + .parallel() + .forEach(model -> { + synchronized (node) { + bySearchImplementationAsDictionary(model.getEntriesSearch(), + model.getType(), + model.isHint() ? EntityType.HINT : EntityType.ENTITY, + node, + model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); + bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), + model.getType(), + EntityType.FALSE_RECOMMENDATION, + node, + model.isDossierDictionary()); + if (model.isDossierDictionary()) { + bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary()); + } + } + }); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index f45527ab..7d09a30c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -13,7 +13,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; -import java.io.Serializable; import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; @@ -33,7 +32,6 @@ import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.amqp.rabbit.core.RabbitTemplate; @@ -45,6 +43,7 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.data.redis.listener.RedisMessageListenerContainer; import org.springframework.test.context.junit.jupiter.SpringExtension; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.common.collect.Sets; @@ -58,7 +57,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; @@ -171,44 +169,12 @@ import lombok.extern.slf4j.Slf4j; ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); -// analyzeService.analyze(analyzeRequest); -// EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); -// -// // Serialize -// String jsonString = objectMapper.writeValueAsString(entityLog); -// // Serialize entityLog to a file -// try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); -// ObjectOutputStream out = new ObjectOutputStream(fileOut)) { -// out.writeObject(jsonString); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// + //analyzeService.analyze(analyzeRequest); + //storeEntityLogAsTempFile(analyzeRequest, objectMapper); + restoreEntityLogFromTempFile(objectMapper, analyzeRequest); - //Deserialize entityLog from a file - String entityLogString = null; - try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) { - entityLogString = (String) in.readObject(); - } catch (IOException | ClassNotFoundException e) { - e.printStackTrace(); - } - var entityLog = objectMapper.readValue(entityLogString, EntityLog.class); - redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); - - ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); - manualRedactionEntry.setAnnotationId("yourAnnotationId"); - manualRedactionEntry.setFileId("fileId"); - manualRedactionEntry.setType("CBI_author"); - manualRedactionEntry.setValue("7232"); - manualRedactionEntry.setReason( - "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)"); - manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); - manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); - manualRedactionEntry.setRequestDate(OffsetDateTime.now()); - manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build())); - - analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build()); + addManualRedactionEntryToFile(analyzeRequest); analyzeService.reanalyze(analyzeRequest); times.add(System.currentTimeMillis() - start); @@ -218,6 +184,53 @@ import lombok.extern.slf4j.Slf4j; } + private static void addManualRedactionEntryToFile(AnalyzeRequest analyzeRequest) { + + ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); + manualRedactionEntry.setAnnotationId("yourAnnotationId"); + manualRedactionEntry.setFileId("fileId"); + manualRedactionEntry.setType("CBI_author"); + manualRedactionEntry.setValue("7232"); + manualRedactionEntry.setReason( + "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)"); + manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); + manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); + manualRedactionEntry.setRequestDate(OffsetDateTime.now()); + manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build())); + + analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build()); + } + + + private void restoreEntityLogFromTempFile(ObjectMapper objectMapper, AnalyzeRequest analyzeRequest) throws JsonProcessingException { + //Deserialize entityLog from a file + String entityLogString = null; + try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) { + entityLogString = (String) in.readObject(); + } catch (IOException | ClassNotFoundException e) { + e.printStackTrace(); + } + var entityLog = objectMapper.readValue(entityLogString, EntityLog.class); + redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + } + + + private void storeEntityLogAsTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException { + + EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + + // Serialize + String jsonString = objectMapper.writeValueAsString(entityLog); + // Serialize entityLog to a file + try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); + ObjectOutputStream out = new ObjectOutputStream(fileOut)) { + out.writeObject(jsonString); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @BeforeEach public void setup() {