From a4988002cd40c9ae396a0ea32e107b2ceb0bead0 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Wed, 27 Mar 2024 14:47:10 +0100 Subject: [PATCH] RED-8702: Explore document databases to store entityLog * fix for all but one (some fixes are hotfixes though) --- .../v1/server/service/AnalyzeService.java | 39 +++++++---- .../service/EntityChangeLogService.java | 47 ++++++++++---- .../service/EntityLogCreatorService.java | 65 +++++++++++-------- .../storage/RedactionStorageService.java | 37 +++++++++++ .../manualchanges/PrecursorEntityTest.java | 2 +- 5 files changed, 138 insertions(+), 52 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 87a7c5a3..60dfa597 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -124,7 +124,8 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryIncrement.getDictionaryVersion(), analyzeRequest, - new ArrayList<>(), new ArrayList<>()); + new ArrayList<>(), + new ArrayList<>()); return finalizeAnalysis(analyzeRequest, startTime, @@ -169,8 +170,9 @@ public class AnalyzeService { EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, document, - notFoundManualOrImportedEntries, entityLogWithoutEntries, + notFoundManualOrImportedEntries, + relevantEntityLogEntries, sectionsToReanalyseIds, dictionary.getVersion()); @@ -233,18 +235,18 @@ public class AnalyzeService { nerEntities); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest, - document, - notFoundManualOrImportedEntries, - dictionary.getVersion(), - kieWrapperEntityRules.rulesVersion()); + EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, + document, + notFoundManualOrImportedEntries, + dictionary.getVersion(), + kieWrapperEntityRules.rulesVersion()); - notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries); + notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries); return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, - EntityLogChanges.builder().entityLog(entityLog).build(), + entityLogChanges, document, document.getNumberOfPages(), dictionary.getVersion(), @@ -265,10 +267,23 @@ public class AnalyzeService { EntityLog entityLog = entityLogChanges.getEntityLog(); - redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog()); + //hotfix for tests (todo 8702) + if (entityLog.getAnalysisNumber() <= 0 && !redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) { + redactionStorageService.insertEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + + } else { + redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog); + + if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) { + redactionStorageService.insertEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries()); + } + if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) { + redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries()); + } + } log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - if (entityLogChanges.isHasChanges() || !isReanalysis) { + if (entityLogChanges.hasChanges() || !isReanalysis) { computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion); } @@ -283,7 +298,7 @@ public class AnalyzeService { .fileId(analyzeRequest.getFileId()) .duration(duration) .numberOfPages(numberOfPages) - .hasUpdates(entityLogChanges.isHasChanges()) + .hasUpdates(entityLogChanges.hasChanges()) .analysisVersion(redactionServiceSettings.getAnalysisVersion()) .analysisNumber(analyzeRequest.getAnalysisNumber()) .rulesVersion(entityLog.getRulesVersion()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index 793ee18c..2225a732 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -1,7 +1,7 @@ package com.iqser.red.service.redaction.v1.server.service; import java.time.OffsetDateTime; -import java.util.Comparator; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; @@ -13,9 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -29,41 +26,58 @@ import lombok.extern.slf4j.Slf4j; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class EntityChangeLogService { + //EntityLogMongoService entityLogMongoService; + + @Timed("redactmanager_computeChanges") - public boolean computeChanges(List previousEntityLogEntries, List newEntityLogEntries, int analysisNumber) { + public EntryChanges computeChanges(//String dossierId, + //String fileId, + List previousEntityLogEntries, + List newEntityLogEntries, + int analysisNumber) { var now = OffsetDateTime.now(); if (previousEntityLogEntries.isEmpty()) { newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now))); - return true; + //entityLogMongoService.insertEntityLogEntries(dossierId, fileId, newEntityLogEntries); + return new EntryChanges(newEntityLogEntries, new ArrayList<>()); } - boolean hasChanges = false; - + List toInsert = new ArrayList<>(); + List toUpdate = new ArrayList<>(); for (EntityLogEntry entityLogEntry : newEntityLogEntries) { Optional optionalPreviousEntity = previousEntityLogEntries.stream() .filter(entry -> entry.getId().equals(entityLogEntry.getId())) .findAny(); if (optionalPreviousEntity.isEmpty()) { - hasChanges = true; entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)); + toInsert.add(entityLogEntry); continue; } EntityLogEntry previousEntity = optionalPreviousEntity.get(); entityLogEntry.getChanges().addAll(previousEntity.getChanges()); if (!previousEntity.getState().equals(entityLogEntry.getState())) { - hasChanges = true; ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState()); entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now)); + toUpdate.add(entityLogEntry); } } - addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now); - return hasChanges; + //entityLogMongoService.insertEntityLogEntries(dossierId, fileId, toInsert); + //entityLogMongoService.updateEntityLogEntries(dossierId, fileId, toUpdate); + + toUpdate.addAll(addRemovedEntriesAsRemoved(//dossierId, fileId, + previousEntityLogEntries, newEntityLogEntries, analysisNumber, now)); + return new EntryChanges(toInsert, toUpdate); } - private void addRemovedEntriesAsRemoved(List previousEntityLogEntries, List newEntityLogEntries, int analysisNumber, OffsetDateTime now) { + private List addRemovedEntriesAsRemoved(//String dossierId, + //String fileId, + List previousEntityLogEntries, + List newEntityLogEntries, + int analysisNumber, + OffsetDateTime now) { Set existingIds = newEntityLogEntries.stream() .map(EntityLogEntry::getId) @@ -73,7 +87,9 @@ public class EntityChangeLogService { .toList(); removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))); removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED)); + //entityLogMongoService.updateEntityLogEntries(dossierId, fileId, removedEntries); newEntityLogEntries.addAll(removedEntries); + return removedEntries; } @@ -97,4 +113,9 @@ public class EntityChangeLogService { return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED)); } + + public record EntryChanges(List inserted, List updated) { + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index c6da211a..04646090 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -32,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import lombok.AccessLevel; @@ -58,11 +59,11 @@ public class EntityLogCreatorService { } - public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, - Document document, - List notFoundEntities, - DictionaryVersion dictionaryVersion, - long rulesVersion) { + public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest, + Document document, + List notFoundEntities, + DictionaryVersion dictionaryVersion, + long rulesVersion) { List entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities); @@ -70,16 +71,24 @@ public class EntityLogCreatorService { List previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); + EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), + //analyzeRequest.getFileId(), + previousExistingEntityLogEntries, + entityLogEntries, + analyzeRequest.getAnalysisNumber()); - return new EntityLog(redactionServiceSettings.getAnalysisVersion(), - analyzeRequest.getAnalysisNumber(), - entityLogEntries, - toEntityLogLegalBasis(legalBasis), - dictionaryVersion.getDossierTemplateVersion(), - dictionaryVersion.getDossierVersion(), - rulesVersion, - legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + return EntityLogChanges.builder() + .entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(), + analyzeRequest.getAnalysisNumber(), + entityLogEntries, + toEntityLogLegalBasis(legalBasis), + dictionaryVersion.getDossierTemplateVersion(), + dictionaryVersion.getDossierVersion(), + rulesVersion, + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()))) + .updatedEntityLogEntries(entryChanges.updated()) + .newEntityLogEntries(entryChanges.inserted()) + .build(); } @@ -93,7 +102,11 @@ public class EntityLogCreatorService { } - public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, List newEntries, List updatedEntries) { + public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, + DictionaryVersion dictionaryVersion, + AnalyzeRequest analyzeRequest, + List newEntries, + List updatedEntries) { List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); @@ -108,8 +121,9 @@ public class EntityLogCreatorService { public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, Document document, + EntityLog entityLogWithoutEntries, List notFoundEntries, - EntityLog previousEntityLog, + List relevantEntityLogEntries, Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { @@ -121,18 +135,17 @@ public class EntityLogCreatorService { .map(EntityLogEntry::getId) .collect(Collectors.toSet()); - List previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() - .stream() - .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() - .get(0)))) - .collect(Collectors.toList()); - previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); + List previousEntriesFromReAnalyzedSections = new ArrayList<>(relevantEntityLogEntries); + previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), newEntityIds)); + previousEntriesFromReAnalyzedSections.addAll(redactionStorageService.findEntriesWithoutContainingNodeIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); + EntryChanges entryChanges = entityChangeLogService.computeChanges(//analyzeRequest.getDossierId(), + //analyzeRequest.getFileId(), + previousEntriesFromReAnalyzedSections, + newEntityLogEntries, + analyzeRequest.getAnalysisNumber()); - previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); - - return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges); + return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 572ad72f..99e51913 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -87,6 +87,34 @@ public class RedactionStorageService { } + @SneakyThrows + public void updateEntityLogWithoutEntries(String dossierId, String fileId, EntityLog entityLog) { + + entityLogMongoService.saveEntityLogWithoutEntries(dossierId, fileId, entityLog); + } + + + @SneakyThrows + public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) { + + entityLogMongoService.insertEntityLog(dossierId, fileId, entityLog); + } + + + @SneakyThrows + public void insertEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + entityLogMongoService.insertEntityLogEntries(dossierId, fileId, entityLogEntries); + } + + + @SneakyThrows + public void updateEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + entityLogMongoService.updateEntityLogEntries(dossierId, fileId, entityLogEntries); + } + + @Timed("redactmanager_getImportedRedactions") public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { @@ -177,6 +205,15 @@ public class RedactionStorageService { return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); } + public List findEntriesWithoutContainingNodeIds(String dossierId, String fileId) { + + return entityLogMongoService.findAllEntityLogEntriesWithoutContainingNodeIds(dossierId, fileId); + } + + public List findEntriesWithIds(String dossierId, String fileId, Collection entryIds) { + + return entityLogMongoService.findAllEntityLogEntriesWithContainingNodeIdsWithEntryIdsIn(dossierId, fileId, entryIds); + } // !Warning! before activating redis cache you need to set diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index b6a1efcb..81acfe52 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -138,7 +138,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { document, notFoundManualEntities, new DictionaryVersion(), - 0L).getEntityLogEntry(); + 0L).getEntityLog().getEntityLogEntry(); assertEquals(1, redactionLogEntries.size()); assertEquals(value, redactionLogEntries.get(0).getValue());