From fc822da032b9d650e4f11479b620c77ec36252bc Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Wed, 27 Mar 2024 17:34:54 +0100 Subject: [PATCH] RED-8702: Explore document databases to store entityLog * fix for test failing due to bson npe (wrong projection) --- .../v1/server/service/AnalyzeService.java | 15 +++++---------- .../service/EntityLogCreatorService.java | 18 +++++------------- .../service/document/SectionFinderService.java | 3 --- .../storage/RedactionStorageService.java | 15 +++++++-------- 4 files changed, 17 insertions(+), 34 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 60dfa597..0c630d19 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -105,22 +105,20 @@ public class AnalyzeService { Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); - List relevantEntityLogEntries = redactionStorageService.findEntriesWithContainingNodeIds(analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - relevantManuallyModifiedAnnotationIds); - Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, + Set sectionsToReanalyseIds = redactionStorageService.findIdsOfSectionsToReanalyse(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + relevantManuallyModifiedAnnotationIds); + sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest, document, dictionaryIncrement, importedRedactions, - relevantEntityLogEntries, - relevantManuallyModifiedAnnotationIds); + relevantManuallyModifiedAnnotationIds)); List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { - // todo: 8702 - this feels wrong EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryIncrement.getDictionaryVersion(), analyzeRequest, @@ -172,7 +170,6 @@ public class AnalyzeService { document, entityLogWithoutEntries, notFoundManualOrImportedEntries, - relevantEntityLogEntries, sectionsToReanalyseIds, dictionary.getVersion()); @@ -351,14 +348,12 @@ public class AnalyzeService { Document document, DictionaryIncrement dictionaryIncrement, ImportedRedactions importedRedactions, - List entityLogEntries, Set relevantManuallyModifiedAnnotationIds) { return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, - entityLogEntries, relevantManuallyModifiedAnnotationIds); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 04646090..b2f092d8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -73,9 +73,7 @@ public class EntityLogCreatorService { EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), //analyzeRequest.getFileId(), - previousExistingEntityLogEntries, - entityLogEntries, - analyzeRequest.getAnalysisNumber()); + previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); return EntityLogChanges.builder() .entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(), @@ -123,7 +121,6 @@ public class EntityLogCreatorService { Document document, EntityLog entityLogWithoutEntries, List notFoundEntries, - List relevantEntityLogEntries, Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { @@ -131,19 +128,14 @@ public class EntityLogCreatorService { .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() .get(0))) .collect(Collectors.toList()); - Set newEntityIds = newEntityLogEntries.stream() - .map(EntityLogEntry::getId) - .collect(Collectors.toSet()); - List previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries); - previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds)); - previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + List previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(), + analyzeRequest.getFileId(), + sectionsToReanalyseIds); EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), //analyzeRequest.getFileId(), - previousEntriesFromReAnalyzedSections, - newEntityLogEntries, - analyzeRequest.getAnalysisNumber()); + previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index aabccc20..f45a3299 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -46,14 +46,11 @@ public class SectionFinderService { Document document, AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, - List entityLogEntries, Set relevantManuallyModifiedAnnotationIds) { long start = System.currentTimeMillis(); Set sectionsToReanalyse = new HashSet<>(); - entityLogEntries.forEach(entityLogEntry -> sectionsToReanalyse.add(entityLogEntry.getContainingNodeId() - .get(0))); var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues() .stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 58904855..f35c7b02 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -5,6 +5,7 @@ import java.io.FileInputStream; import java.io.InputStream; import java.util.Collection; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import org.springframework.cache.annotation.Cacheable; @@ -201,21 +202,19 @@ public class RedactionStorageService { } - public List findEntriesWithContainingNodeIds(String dossierId, String fileId, Collection entryIds) { + public Set findIdsOfSectionsToReanalyse(String dossierId, String fileId, Collection entryIds) { - return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); - } - public List findEntriesWithoutContainingNodeIds(String dossierId, String fileId) { - - return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId); + return entityLogMongoService.findFirstContainingNodeIdForEachEntry(dossierId, fileId, entryIds); } - public List findEntriesWithIds(String dossierId, String fileId, Collection entryIds) { - return entityLogMongoService.findAllEntityLogEntriesWithEntryIdsIn(dossierId, fileId, entryIds); + public List findEntriesContainedBySectionsOrNotContained(String dossierId, String fileId, Collection sectionIds) { + + return entityLogMongoService.findEntityLogEntriesNotContainedOrFirstContainedByElementInList(dossierId, fileId, sectionIds); } + // !Warning! before activating redis cache you need to set // -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M // Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb.