From af791f13eb31a2102ee3e3f0215b29e88fe72775 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Mon, 18 Mar 2024 15:05:41 +0100 Subject: [PATCH] RED-8702: Explore document databases to store entityLog * added mongodb documents, repositories and EntityLogMongoService * refactored places where entity logs were stored and loaded to use new logic --- .../persistence/EntityLogMongoService.java | 238 ++++++++++++++++++ .../EntityLogDocumentRepository.java | 15 ++ .../EntityLogEntryDocumentRepository.java | 23 ++ .../build.gradle.kts | 1 - 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntityLogMongoService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogDocumentRepository.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogEntryDocumentRepository.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntityLogMongoService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntityLogMongoService.java new file mode 100644 index 000000000..f398ddf69 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntityLogMongoService.java @@ -0,0 +1,238 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument; +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument; +import com.iqser.red.service.persistence.management.v1.processor.exception.EntityLogDocumentNotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntityLogDocumentRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntityLogEntryDocumentRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; + +@Service +public class EntityLogMongoService { + + private final EntityLogDocumentRepository entityLogDocumentRepository; + private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository; + + + public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository, EntityLogEntryDocumentRepository entityLogEntryDocumentRepository) { + + this.entityLogDocumentRepository = entityLogDocumentRepository; + this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository; + } + + + public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) { + + EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog)); + + entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry() + .stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry)) + .toList()); + } + + + // this does everything : insert when not found and update if found + // todo: remove and replace when services use insert,update,delete correctly + public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) { + + Optional optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId)); + if (optionalEntityLogDocument.isEmpty()) { + // throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId)); + insertEntityLog(dossierId, fileId, entityLog); + return; + } + + EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get(); + List oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument(); + + EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog); + List newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument(); + + List toUpdate = new ArrayList<>(oldEntityLogEntryDocuments); + toUpdate.retainAll(newEntityLogEntryDocuments); + + List toRemove = new ArrayList<>(oldEntityLogEntryDocuments); + toRemove.removeAll(toUpdate); + + List toInsert = new ArrayList<>(newEntityLogEntryDocuments); + toInsert.removeAll(toUpdate); + + entityLogEntryDocumentRepository.saveAll(toUpdate); + entityLogEntryDocumentRepository.deleteAll(toRemove); + entityLogEntryDocumentRepository.insert(toInsert); + + entityLogDocumentRepository.save(newEntityLogDocument); + } + + + public void deleteEntityLog(String dossierId, String fileId) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + entityLogDocumentRepository.deleteById(entityLogId); + + entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId); + } + + + public void insertEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId); + + List entityLogEntryDocuments = entityLogEntries.stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry)) + .toList(); + + entityLogDocument.getEntityLogEntryDocument().addAll(entityLogEntryDocuments); + + entityLogEntryDocumentRepository.insert(entityLogEntryDocuments); + entityLogDocumentRepository.save(entityLogDocument); + } + + + public void updateEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry)) + .toList()); + } + + public void deleteEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId); + + List entityLogEntryDocuments = entityLogEntries.stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry)) + .toList(); + + entityLogDocument.getEntityLogEntryDocument().removeAll(entityLogEntryDocuments); + + entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments); + entityLogDocumentRepository.save(entityLogDocument); + } + + + private EntityLogDocument getEntityLogDocument(String entityLogId) { + + Optional optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId); + + if (optionalEntityLogDocument.isEmpty()) { + throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId)); + } + + return optionalEntityLogDocument.get(); + } + + + public Optional findEntityLogByDossierIdAndFileId(String dossierId, String fileId) { + + return entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId)) + .map(EntityLogMongoService::fromDocument); + } + + + public boolean entityLogDocumentExists(String dossierId, String fileId) { + + return entityLogDocumentRepository.existsById(EntityLogDocument.getDocumentId(dossierId, fileId)); + } + + + public Optional findLatestAnalysisNumber(String dossierId, String fileId) { + + return entityLogDocumentRepository.findAnalysisNumberById(EntityLogDocument.getDocumentId(dossierId, fileId)) + .map(EntityLogDocument::getAnalysisNumber); + } + + + public List findAllEntityLogEntriesWithManualChanges(String dossierId, String fileId) { + + return entityLogEntryDocumentRepository.findByEntityLogIdAndManualChangesNotEmpty(EntityLogDocument.getDocumentId(dossierId, fileId)) + .stream() + .map(EntityLogMongoService::fromDocument) + .toList(); + } + + + public List findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) { + + return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(EntityLogDocument.getDocumentId(dossierId, fileId), analysisNumber) + .stream() + .map(EntityLogMongoService::fromDocument) + .toList(); + } + + + public List findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) { + + return entityLogEntryDocumentRepository.findByEntityLogId(EntityLogDocument.getDocumentId(dossierId, fileId)) + .stream() + .map(EntityLogMongoService::fromDocument) + .toList(); + } + + + private static EntityLog fromDocument(EntityLogDocument entityLogDocument) { + + EntityLog entityLog = new EntityLog(); + entityLog.setAnalysisVersion(entityLogDocument.getAnalysisVersion()); + entityLog.setAnalysisNumber(entityLogDocument.getAnalysisNumber()); + entityLog.setEntityLogEntry(entityLogDocument.getEntityLogEntryDocument() + .stream() + .map(EntityLogMongoService::fromDocument) + .collect(Collectors.toList())); + entityLog.setLegalBasis(entityLogDocument.getLegalBasis()); + entityLog.setDictionaryVersion(entityLogDocument.getDictionaryVersion()); + entityLog.setDossierDictionaryVersion(entityLogDocument.getDossierDictionaryVersion()); + entityLog.setRulesVersion(entityLogDocument.getRulesVersion()); + entityLog.setLegalBasisVersion(entityLogDocument.getLegalBasisVersion()); + return entityLog; + } + + + private static EntityLogEntry fromDocument(EntityLogEntryDocument entityLogEntryDocument) { + + EntityLogEntry entityLogEntry = new EntityLogEntry(); + entityLogEntry.setId(entityLogEntryDocument.getEntryId()); + entityLogEntry.setState(entityLogEntryDocument.getState()); + entityLogEntry.setValue(entityLogEntryDocument.getValue()); + entityLogEntry.setReason(entityLogEntryDocument.getReason()); + entityLogEntry.setMatchedRule(entityLogEntryDocument.getMatchedRule()); + entityLogEntry.setLegalBasis(entityLogEntryDocument.getLegalBasis()); + entityLogEntry.setContainingNodeId(new ArrayList<>(entityLogEntryDocument.getContainingNodeId())); + entityLogEntry.setClosestHeadline(entityLogEntryDocument.getClosestHeadline()); + entityLogEntry.setSection(entityLogEntryDocument.getSection()); + entityLogEntry.setPositions(new ArrayList<>(entityLogEntryDocument.getPositions())); + entityLogEntry.setTextBefore(entityLogEntryDocument.getTextBefore()); + entityLogEntry.setTextAfter(entityLogEntryDocument.getTextAfter()); + entityLogEntry.setStartOffset(entityLogEntryDocument.getStartOffset()); + entityLogEntry.setEndOffset(entityLogEntryDocument.getEndOffset()); + entityLogEntry.setImageHasTransparency(entityLogEntryDocument.isImageHasTransparency()); + entityLogEntry.setDictionaryEntry(entityLogEntryDocument.isDictionaryEntry()); + entityLogEntry.setDossierDictionaryEntry(entityLogEntryDocument.isDossierDictionaryEntry()); + entityLogEntry.setExcluded(entityLogEntryDocument.isExcluded()); + entityLogEntry.setChanges(new ArrayList<>(entityLogEntryDocument.getChanges())); + entityLogEntry.setManualChanges(new ArrayList<>(entityLogEntryDocument.getManualChanges())); + entityLogEntry.setEngines(new HashSet<>(entityLogEntryDocument.getEngines())); + entityLogEntry.setReference(new HashSet<>(entityLogEntryDocument.getReference())); + entityLogEntry.setImportedRedactionIntersections(new HashSet<>(entityLogEntryDocument.getImportedRedactionIntersections())); + entityLogEntry.setNumberOfComments(entityLogEntryDocument.getNumberOfComments()); + return entityLogEntry; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogDocumentRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogDocumentRepository.java new file mode 100644 index 000000000..81249cfcd --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogDocumentRepository.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.Optional; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.repository.query.Param; + +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument; + +public interface EntityLogDocumentRepository extends MongoRepository { + + @Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }") + Optional findAnalysisNumberById(@Param("_id") String id); +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogEntryDocumentRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogEntryDocumentRepository.java new file mode 100644 index 000000000..8e1d82a06 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntityLogEntryDocumentRepository.java @@ -0,0 +1,23 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; + +import java.util.List; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; + +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument; + +public interface EntityLogEntryDocumentRepository extends MongoRepository { + + @Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }") + List findByEntityLogIdAndManualChangesNotEmpty(String entityLogId); + + @Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }") + List findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber); + + @Query("{ 'entityLogId' : ?0}") + List findByEntityLogId(String entityLogId); + + @Query(value = "{ 'entityLogId' : ?0}", delete = true) + void deleteByEntityLogId(String entityLogId); +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts index 8e4c8efce..b2dfa2a93 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts @@ -11,7 +11,6 @@ dependencies { api("org.springframework.boot:spring-boot-starter-validation:3.1.3") api("com.iqser.red.commons:jackson-commons:2.1.0") api("com.iqser.red.commons:dictionary-merge-commons:1.3.0") - implementation("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}") testImplementation("com.iqser.red.commons:test-commons:2.1.0") testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4") compileOnly("org.springdoc:springdoc-openapi-ui:1.7.0")