RED-8702: Explore document databases to store entityLog
* pre-test eval version RED-8702: Explore document databases to store entityLog * pre-test eval version RED-8702: Explore document databases to store entityLog * refactoring RED-8702: Explore document databases to store entityLog * added new CRUD operations to repositories and services * removed cascade logic as updates and deletes are not trivial to implement, and thus it is better to do all CRUD operations alike RED-8702: Explore document databases to store entityLog * added experimental logic for mongodb integration RED-8702: Explore document databases to store entityLog * added experimental logic for mongodb integration RED-8702 RED-8702 RED-8702
This commit is contained in:
maverickstuder
parent
983f728248
commit
bb8649d3e8
@ -16,7 +16,7 @@ val layoutParserVersion = "0.96.0"
|
||||
val jacksonVersion = "2.15.2"
|
||||
val droolsVersion = "9.44.0.Final"
|
||||
val pdfBoxVersion = "3.0.0"
|
||||
val persistenceServiceVersion = "2.366.0"
|
||||
val persistenceServiceVersion = "maverick-mongo"
|
||||
val springBootStarterVersion = "3.1.5"
|
||||
|
||||
configurations {
|
||||
@ -31,6 +31,7 @@ dependencies {
|
||||
|
||||
implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
|
||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
|
||||
implementation("com.iqser.red.service:persistence-service-processor-v1:${persistenceServiceVersion}")
|
||||
implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
|
||||
|
||||
implementation("com.iqser.red.commons:spring-commons:6.2.0")
|
||||
@ -38,7 +39,8 @@ dependencies {
|
||||
|
||||
implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
|
||||
implementation("com.iqser.red.commons:storage-commons:2.45.0")
|
||||
implementation("com.knecon.fforesight:tenant-commons:0.21.0")
|
||||
implementation("com.knecon.fforesight:tenant-commons:maverick-mongo")
|
||||
implementation("com.knecon.fforesight:mongo-database-commons:maverick-mongo")
|
||||
implementation("com.knecon.fforesight:tracing-commons:0.5.0")
|
||||
|
||||
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
|
||||
@ -56,12 +58,21 @@ dependencies {
|
||||
implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
|
||||
implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
|
||||
implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
|
||||
implementation("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
|
||||
|
||||
implementation("net.logstash.logback:logstash-logback-encoder:7.4")
|
||||
implementation("ch.qos.logback:logback-classic")
|
||||
|
||||
implementation("org.reflections:reflections:0.10.2")
|
||||
|
||||
//todo: remove
|
||||
implementation("io.minio:minio:8.5.9")
|
||||
|
||||
|
||||
implementation("org.liquibase:liquibase-core:4.24.0")
|
||||
|
||||
implementation("org.liquibase.ext:liquibase-mongodb:4.24.0")
|
||||
|
||||
testImplementation(project(":rules-management"))
|
||||
testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
|
||||
testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
|
||||
|
||||
@ -4,16 +4,20 @@ import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
|
||||
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
|
||||
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
|
||||
|
||||
import io.micrometer.core.aop.TimedAspect;
|
||||
@ -23,10 +27,11 @@ import io.micrometer.observation.aop.ObservedAspect;
|
||||
|
||||
@EnableCaching
|
||||
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
|
||||
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class})
|
||||
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
|
||||
@EnableFeignClients(basePackageClasses = RulesClient.class)
|
||||
@EnableConfigurationProperties(RedactionServiceSettings.class)
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
|
||||
@EnableMongoRepositories
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class})
|
||||
public class Application {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
@ -70,7 +70,8 @@ public class MigrationMessageReceiver {
|
||||
migrationRequest.getFileId());
|
||||
|
||||
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
|
||||
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
||||
//todo
|
||||
redactionStorageService.saveEntityLog(migrationRequest.getDossierId(), migrationRequest.getFileId(), migratedEntityLog.getEntityLog());
|
||||
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
|
||||
|
||||
sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.mongo;
|
||||
|
||||
public class EntityLogDocumentNotFoundException extends RuntimeException {
|
||||
|
||||
public EntityLogDocumentNotFoundException(String errorMessage) {
|
||||
|
||||
super(errorMessage);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.mongo;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.data.mongodb.repository.MongoRepository;
|
||||
import org.springframework.data.mongodb.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
|
||||
|
||||
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
|
||||
|
||||
@Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }")
|
||||
Optional<EntityLogDocument> findAnalysisNumberById(@Param("_id") String id);
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package com.iqser.red.service.redaction.v1.server.mongo;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.data.mongodb.repository.MongoRepository;
|
||||
import org.springframework.data.mongodb.repository.Query;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
|
||||
|
||||
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {
|
||||
|
||||
@Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }")
|
||||
List<EntityLogEntryDocument> findByEntityLogIdAndManualChangesNotEmpty(String entityLogId);
|
||||
|
||||
@Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }")
|
||||
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber);
|
||||
|
||||
@Query("{ 'entityLogId' : ?0}")
|
||||
List<EntityLogEntryDocument> findByEntityLogId(String entityLogId);
|
||||
|
||||
@Query(value = "{ 'entityLogId' : ?0}", delete = true)
|
||||
void deleteByEntityLogId(String entityLogId);
|
||||
}
|
||||
@ -0,0 +1,237 @@
|
||||
package com.iqser.red.service.redaction.v1.server.mongo;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
|
||||
@Service
|
||||
public class EntityLogMongoService {
|
||||
|
||||
private final EntityLogDocumentRepository entityLogDocumentRepository;
|
||||
private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
|
||||
|
||||
|
||||
public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository, EntityLogEntryDocumentRepository entityLogEntryDocumentRepository) {
|
||||
|
||||
this.entityLogDocumentRepository = entityLogDocumentRepository;
|
||||
this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
|
||||
}
|
||||
|
||||
|
||||
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
|
||||
|
||||
EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog));
|
||||
|
||||
entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
// this does everything : insert when not found and update if found
|
||||
// todo: remove and replace when services use insert,update,delete correctly
|
||||
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
|
||||
|
||||
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId));
|
||||
if (optionalEntityLogDocument.isEmpty()) {
|
||||
// throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId));
|
||||
insertEntityLog(dossierId, fileId, entityLog);
|
||||
return;
|
||||
}
|
||||
|
||||
EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get();
|
||||
List<EntityLogEntryDocument> oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument();
|
||||
|
||||
EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog);
|
||||
List<EntityLogEntryDocument> newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument();
|
||||
|
||||
List<EntityLogEntryDocument> toUpdate = new ArrayList<>(oldEntityLogEntryDocuments);
|
||||
toUpdate.retainAll(newEntityLogEntryDocuments);
|
||||
|
||||
List<EntityLogEntryDocument> toRemove = new ArrayList<>(oldEntityLogEntryDocuments);
|
||||
toRemove.removeAll(toUpdate);
|
||||
|
||||
List<EntityLogEntryDocument> toInsert = new ArrayList<>(newEntityLogEntryDocuments);
|
||||
toInsert.removeAll(toUpdate);
|
||||
|
||||
entityLogEntryDocumentRepository.saveAll(toUpdate);
|
||||
entityLogEntryDocumentRepository.deleteAll(toRemove);
|
||||
entityLogEntryDocumentRepository.insert(toInsert);
|
||||
|
||||
entityLogDocumentRepository.save(newEntityLogDocument);
|
||||
}
|
||||
|
||||
|
||||
public void deleteEntityLog(String dossierId, String fileId) {
|
||||
|
||||
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
|
||||
|
||||
entityLogDocumentRepository.deleteById(entityLogId);
|
||||
|
||||
entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId);
|
||||
}
|
||||
|
||||
|
||||
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
|
||||
|
||||
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
|
||||
|
||||
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
|
||||
|
||||
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
|
||||
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
|
||||
.toList();
|
||||
|
||||
entityLogDocument.getEntityLogEntryDocument().addAll(entityLogEntryDocuments);
|
||||
|
||||
entityLogEntryDocumentRepository.insert(entityLogEntryDocuments);
|
||||
entityLogDocumentRepository.save(entityLogDocument);
|
||||
}
|
||||
|
||||
|
||||
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
|
||||
|
||||
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
|
||||
|
||||
entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream()
|
||||
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
|
||||
.toList());
|
||||
}
|
||||
|
||||
public void deleteEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
|
||||
|
||||
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
|
||||
|
||||
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
|
||||
|
||||
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
|
||||
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
|
||||
.toList();
|
||||
|
||||
entityLogDocument.getEntityLogEntryDocument().removeAll(entityLogEntryDocuments);
|
||||
|
||||
entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments);
|
||||
entityLogDocumentRepository.save(entityLogDocument);
|
||||
}
|
||||
|
||||
|
||||
@NotNull
|
||||
private EntityLogDocument getEntityLogDocument(String entityLogId) {
|
||||
|
||||
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);
|
||||
|
||||
if (optionalEntityLogDocument.isEmpty()) {
|
||||
throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId));
|
||||
}
|
||||
|
||||
return optionalEntityLogDocument.get();
|
||||
}
|
||||
|
||||
|
||||
public Optional<EntityLog> findEntityLogByDossierIdAndFileId(String dossierId, String fileId) {
|
||||
|
||||
return entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId))
|
||||
.map(EntityLogMongoService::fromDocument);
|
||||
}
|
||||
|
||||
|
||||
public boolean entityLogDocumentExists(String dossierId, String fileId) {
|
||||
|
||||
return entityLogDocumentRepository.existsById(EntityLogDocument.getDocumentId(dossierId, fileId));
|
||||
}
|
||||
|
||||
|
||||
public Optional<Integer> findLatestAnalysisNumber(String dossierId, String fileId) {
|
||||
|
||||
return entityLogDocumentRepository.findAnalysisNumberById(EntityLogDocument.getDocumentId(dossierId, fileId))
|
||||
.map(EntityLogDocument::getAnalysisNumber);
|
||||
}
|
||||
|
||||
|
||||
public List<EntityLogEntry> findAllEntityLogEntriesWithManualChanges(String dossierId, String fileId) {
|
||||
|
||||
return entityLogEntryDocumentRepository.findByEntityLogIdAndManualChangesNotEmpty(EntityLogDocument.getDocumentId(dossierId, fileId))
|
||||
.stream()
|
||||
.map(EntityLogMongoService::fromDocument)
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
public List<EntityLogEntry> findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) {
|
||||
|
||||
return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(EntityLogDocument.getDocumentId(dossierId, fileId), analysisNumber)
|
||||
.stream()
|
||||
.map(EntityLogMongoService::fromDocument)
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
public List<EntityLogEntry> findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) {
|
||||
|
||||
return entityLogEntryDocumentRepository.findByEntityLogId(EntityLogDocument.getDocumentId(dossierId, fileId))
|
||||
.stream()
|
||||
.map(EntityLogMongoService::fromDocument)
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
private static EntityLog fromDocument(EntityLogDocument entityLogDocument) {
|
||||
|
||||
EntityLog entityLog = new EntityLog();
|
||||
entityLog.setAnalysisVersion(entityLogDocument.getAnalysisVersion());
|
||||
entityLog.setAnalysisNumber(entityLogDocument.getAnalysisNumber());
|
||||
entityLog.setEntityLogEntry(entityLogDocument.getEntityLogEntryDocument()
|
||||
.stream()
|
||||
.map(EntityLogMongoService::fromDocument)
|
||||
.collect(Collectors.toList()));
|
||||
entityLog.setLegalBasis(entityLogDocument.getLegalBasis());
|
||||
entityLog.setDictionaryVersion(entityLogDocument.getDictionaryVersion());
|
||||
entityLog.setDossierDictionaryVersion(entityLogDocument.getDossierDictionaryVersion());
|
||||
entityLog.setRulesVersion(entityLogDocument.getRulesVersion());
|
||||
entityLog.setLegalBasisVersion(entityLogDocument.getLegalBasisVersion());
|
||||
return entityLog;
|
||||
}
|
||||
|
||||
|
||||
private static EntityLogEntry fromDocument(EntityLogEntryDocument entityLogEntryDocument) {
|
||||
|
||||
EntityLogEntry entityLogEntry = new EntityLogEntry();
|
||||
entityLogEntry.setId(entityLogEntryDocument.getEntryId());
|
||||
entityLogEntry.setState(entityLogEntryDocument.getState());
|
||||
entityLogEntry.setValue(entityLogEntryDocument.getValue());
|
||||
entityLogEntry.setReason(entityLogEntryDocument.getReason());
|
||||
entityLogEntry.setMatchedRule(entityLogEntryDocument.getMatchedRule());
|
||||
entityLogEntry.setLegalBasis(entityLogEntryDocument.getLegalBasis());
|
||||
entityLogEntry.setContainingNodeId(new ArrayList<>(entityLogEntryDocument.getContainingNodeId()));
|
||||
entityLogEntry.setClosestHeadline(entityLogEntryDocument.getClosestHeadline());
|
||||
entityLogEntry.setSection(entityLogEntryDocument.getSection());
|
||||
entityLogEntry.setPositions(new ArrayList<>(entityLogEntryDocument.getPositions()));
|
||||
entityLogEntry.setTextBefore(entityLogEntryDocument.getTextBefore());
|
||||
entityLogEntry.setTextAfter(entityLogEntryDocument.getTextAfter());
|
||||
entityLogEntry.setStartOffset(entityLogEntryDocument.getStartOffset());
|
||||
entityLogEntry.setEndOffset(entityLogEntryDocument.getEndOffset());
|
||||
entityLogEntry.setImageHasTransparency(entityLogEntryDocument.isImageHasTransparency());
|
||||
entityLogEntry.setDictionaryEntry(entityLogEntryDocument.isDictionaryEntry());
|
||||
entityLogEntry.setDossierDictionaryEntry(entityLogEntryDocument.isDossierDictionaryEntry());
|
||||
entityLogEntry.setExcluded(entityLogEntryDocument.isExcluded());
|
||||
entityLogEntry.setChanges(new ArrayList<>(entityLogEntryDocument.getChanges()));
|
||||
entityLogEntry.setManualChanges(new ArrayList<>(entityLogEntryDocument.getManualChanges()));
|
||||
entityLogEntry.setEngines(new HashSet<>(entityLogEntryDocument.getEngines()));
|
||||
entityLogEntry.setReference(new HashSet<>(entityLogEntryDocument.getReference()));
|
||||
entityLogEntry.setImportedRedactionIntersections(new HashSet<>(entityLogEntryDocument.getImportedRedactionIntersections()));
|
||||
entityLogEntry.setNumberOfComments(entityLogEntryDocument.getNumberOfComments());
|
||||
return entityLogEntry;
|
||||
}
|
||||
|
||||
}
|
||||
@ -255,7 +255,8 @@ public class AnalyzeService {
|
||||
Set<FileAttribute> addedFileAttributes) {
|
||||
|
||||
EntityLog entityLog = entityLogChanges.getEntityLog();
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
|
||||
//redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
|
||||
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getEntityLog());
|
||||
|
||||
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
if (entityLogChanges.isHasChanges() || !isReanalysis) {
|
||||
|
||||
@ -10,13 +10,14 @@ import org.springframework.cache.annotation.Cacheable;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogMongoService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
@ -39,6 +40,8 @@ public class RedactionStorageService {
|
||||
|
||||
private final StorageService storageService;
|
||||
|
||||
private final EntityLogMongoService entityLogMongoService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public InputStream getStoredObject(String storageId) {
|
||||
@ -75,6 +78,13 @@ public class RedactionStorageService {
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
|
||||
|
||||
entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getImportedRedactions")
|
||||
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
|
||||
|
||||
@ -132,7 +142,8 @@ public class RedactionStorageService {
|
||||
public EntityLog getEntityLog(String dossierId, String fileId) {
|
||||
|
||||
try {
|
||||
EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
|
||||
//EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
|
||||
EntityLog entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId).orElseThrow(() -> new StorageObjectDoesNotExist(""));
|
||||
entityLog.setEntityLogEntry(entityLog.getEntityLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
|
||||
@ -200,7 +211,7 @@ public class RedactionStorageService {
|
||||
|
||||
public boolean entityLogExists(String dossierId, String fileId) {
|
||||
|
||||
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG));
|
||||
return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -46,6 +46,14 @@ spring:
|
||||
port: ${REDIS_PORT:6379}
|
||||
password: ${REDIS_PASSWORD}
|
||||
timeout: 60000
|
||||
mongodb:
|
||||
auto-index-creation: true
|
||||
# todo: multi-tenancy
|
||||
database: redaction
|
||||
host: ${MONGODB_HOST:localhost}
|
||||
port: 27017
|
||||
username: ${MONGODB_USER}
|
||||
password: ${MONGODB_PASSWORD}
|
||||
|
||||
management:
|
||||
endpoint:
|
||||
|
||||
@ -179,7 +179,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
|
||||
manualRedactions,
|
||||
TEST_FILE_ID);
|
||||
|
||||
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
||||
redactionStorageService.saveEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID, migratedEntityLog.getEntityLog());
|
||||
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
|
||||
EntityLog entityLog = migratedEntityLog.getEntityLog();
|
||||
assertEquals(mergedRedactionLog.getAnalysisNumber(), entityLog.getAnalysisNumber());
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static com.mongodb.client.model.Filters.eq;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import liquibase.Contexts;
|
||||
import liquibase.Liquibase;
|
||||
import liquibase.database.DatabaseFactory;
|
||||
import liquibase.ext.mongodb.database.MongoLiquibaseDatabase;
|
||||
import liquibase.resource.ClassLoaderResourceAccessor;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
public class MyMongoLiquibaseTest {
|
||||
|
||||
private static final String HOSTNAME = "localhost";
|
||||
private static final String PORT = "27017";
|
||||
private static final String DATABASE = "redaction";
|
||||
private static final String USERNAME = "root";
|
||||
private static final String PASSWORD = "AP8ckozpAl55JDAAFR4rxpbPj1WXdBTL";
|
||||
|
||||
private static final String URL = String.format("mongodb://%s:%s/%s", HOSTNAME, PORT, DATABASE);
|
||||
|
||||
@Test
|
||||
@Disabled
|
||||
@SneakyThrows
|
||||
public void testMongoDB() {
|
||||
|
||||
try (MongoLiquibaseDatabase database = (MongoLiquibaseDatabase) DatabaseFactory.getInstance().openDatabase(URL, USERNAME, PASSWORD, null, null)) {
|
||||
Liquibase liquibase = new Liquibase("mongo/liquibase/test.changelog.xml", new ClassLoaderResourceAccessor(), database);
|
||||
liquibase.update();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,102 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static com.mongodb.client.model.Filters.eq;
|
||||
|
||||
import org.bson.BsonArray;
|
||||
import org.bson.BsonDocument;
|
||||
import org.bson.BsonString;
|
||||
import org.bson.Document;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.data.mongodb.core.MongoTemplate;
|
||||
|
||||
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
|
||||
import com.mongodb.client.MongoClient;
|
||||
import com.mongodb.client.MongoClients;
|
||||
import com.mongodb.client.MongoCollection;
|
||||
import com.mongodb.client.MongoDatabase;
|
||||
|
||||
@Disabled
|
||||
public class MyMongoTest {
|
||||
|
||||
private static final String HOSTNAME = "localhost";
|
||||
private static final String PORT = "27017";
|
||||
private static final String DATABASE = "redaction";
|
||||
private static final String USERNAME = "root";
|
||||
private static final String PASSWORD = "AP8ckozpAl55JDAAFR4rxpbPj1WXdBTL";
|
||||
|
||||
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
|
||||
public static final String COLLECTION_ENTITY_LOG = "entity-logs";
|
||||
|
||||
|
||||
private static final String CONNECTION_STRING = String.format("mongodb://%s:%s@%s:%s/", USERNAME, PASSWORD, HOSTNAME, PORT);
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void deleteMongoDBDatabase() {
|
||||
|
||||
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
|
||||
USERNAME,
|
||||
PASSWORD,
|
||||
HOSTNAME,
|
||||
PORT))) {
|
||||
mongoClient.getDatabase("new_database").drop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void createDatabaseUser() {
|
||||
|
||||
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
|
||||
USERNAME,
|
||||
PASSWORD,
|
||||
HOSTNAME,
|
||||
PORT))) {
|
||||
MongoDatabase database = mongoClient.getDatabase("last_database");
|
||||
BsonDocument createUserCommand = new BsonDocument();
|
||||
createUserCommand.append("createUser", new BsonString(USERNAME));
|
||||
createUserCommand.append("pwd", new BsonString(PASSWORD));
|
||||
BsonArray roles = new BsonArray();
|
||||
roles.add(new BsonString("readWrite"));
|
||||
createUserCommand.append("roles", roles);
|
||||
|
||||
database.runCommand(createUserCommand);
|
||||
|
||||
}
|
||||
}
|
||||
@Test
|
||||
public void testMongoDB() {
|
||||
|
||||
try (MongoClient mongoClient = MongoClients.create(CONNECTION_STRING)) {
|
||||
MongoDatabase database = mongoClient.getDatabase(DATABASE);
|
||||
MongoCollection<Document> collection = database.getCollection(COLLECTION_ENTITY_LOG);
|
||||
Document doc = collection.find(eq("_id", TEST_FILE_ID)).first();
|
||||
if (doc != null) {
|
||||
System.out.println("Found doc with _id: " + doc.get("_id"));
|
||||
} else {
|
||||
System.out.println("No matching documents found.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMongoDB_Template() {
|
||||
|
||||
try (MongoClient mongoClient = MongoClients.create(CONNECTION_STRING)) {
|
||||
MongoTemplate mongoTemplate = new MongoTemplate(mongoClient, DATABASE);
|
||||
|
||||
MongoCollection<Document> collection = mongoTemplate.getCollection(COLLECTION_ENTITY_LOG);
|
||||
Document doc = collection.find(eq("_id", TEST_FILE_ID)).first();
|
||||
if (doc != null) {
|
||||
System.out.println("Found doc with _id: " + doc.get("_id"));
|
||||
} else {
|
||||
System.out.println("No matching documents found.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,442 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.boot.test.util.TestPropertyValues;
|
||||
import org.springframework.context.ApplicationContextInitializer;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.FilterType;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogMongoService;
|
||||
import com.iqser.red.storage.commons.service.StorageServiceImpl;
|
||||
import com.knecon.fforesight.mongo.database.commons.config.MongoDbConfiguration;
|
||||
import com.knecon.fforesight.mongo.database.commons.config.MultiTenantMongoDBFactory;
|
||||
import com.knecon.fforesight.mongo.database.commons.service.MongoClientCache;
|
||||
import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
|
||||
import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProviderImpl;
|
||||
import com.knecon.fforesight.mongo.database.commons.service.MongoDataSources;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
import com.knecon.fforesight.tenantcommons.TenantsClient;
|
||||
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
|
||||
|
||||
import io.minio.MinioClient;
|
||||
import io.minio.PutObjectArgs;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Import(MyOtherMongoTest.MongoTestConfiguration.class)
|
||||
@ContextConfiguration(initializers = {MyOtherMongoTest.Initializer.class})
|
||||
@Disabled
|
||||
public class MyOtherMongoTest {
|
||||
|
||||
@MockBean
|
||||
private RabbitTemplate rabbitTemplate;
|
||||
|
||||
@MockBean
|
||||
private TenantsClient tenantsClient;
|
||||
|
||||
@MockBean
|
||||
protected KieContainer kieContainer;
|
||||
|
||||
@MockBean
|
||||
protected DictionaryClient dictionaryClient;
|
||||
|
||||
private static final String HOSTNAME = "localhost";
|
||||
private static final String PORT = "27017";
|
||||
private static final String DATABASE = "redaction";
|
||||
private static final String USERNAME = "root";
|
||||
private static final String PASSWORD = "AP8ckozpAl55JDAAFR4rxpbPj1WXdBTL";
|
||||
|
||||
private static final String TEST_DOSSIER_2_ID = "bigentestdossierid";
|
||||
private static final String TEST_FILE_2_ID = "bigentityfileid";
|
||||
private static final String TEST_FILE_3_ID = "upd_bigentityfileid";
|
||||
private static final String TEST_DOSSIER_ID = "testdossierid";
|
||||
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
|
||||
public static final String COLLECTION_ENTITY_LOG = "entity-logs";
|
||||
|
||||
private static final String CONNECTION_STRING = String.format("mongodb://%s:%s@%s:%s/", USERNAME, PASSWORD, HOSTNAME, PORT);
|
||||
|
||||
@Autowired
|
||||
private StorageServiceImpl storageService;
|
||||
|
||||
@Autowired
|
||||
private EntityLogMongoService entityLogMongoService;
|
||||
|
||||
@MockBean
|
||||
private MongoConnectionProvider mongoConnectionProvider;
|
||||
|
||||
|
||||
@BeforeEach
|
||||
public void setupTenantAndMockDBConnection() {
|
||||
|
||||
TenantContext.setTenantId("redaction");
|
||||
|
||||
when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder()
|
||||
.host("localhost")
|
||||
.port("27017")
|
||||
.database("redaction")
|
||||
.username(USERNAME)
|
||||
.password(PASSWORD)
|
||||
.build());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testUpdateBigEntityLogDocumentByAddingOne() {
|
||||
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
|
||||
EntityLogEntry entityLogEntry = objectMapper.readValue(NEW_ENTITY_LOG_ENTRY, EntityLogEntry.class);
|
||||
|
||||
entityLogMongoService.insertEntityLogEntries(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, List.of(entityLogEntry));
|
||||
|
||||
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
|
||||
assertTrue(found.isPresent());
|
||||
assertEquals(found.get().getEntityLogEntry().size(), 1707);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testUpdateBigEntityLogDocumentByNewLog() {
|
||||
|
||||
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_3_ID));
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
|
||||
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
|
||||
entityLog.setAnalysisNumber(entityLog.getAnalysisNumber() + 1);
|
||||
|
||||
entityLogMongoService.saveEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
|
||||
|
||||
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
|
||||
assertTrue(found.isPresent());
|
||||
assertEquals(found.get().getAnalysisNumber(), entityLog.getAnalysisNumber());
|
||||
assertEquals(found.get().getEntityLogEntry().size(), 1707);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testUpdateBigEntityLogDocumentAnalysisNumber() {
|
||||
|
||||
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_2_ID));
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
|
||||
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
|
||||
entityLog.setAnalysisNumber(entityLog.getAnalysisNumber() + 1);
|
||||
|
||||
entityLogMongoService.saveEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
|
||||
|
||||
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
|
||||
assertTrue(found.isPresent());
|
||||
assertEquals(found.get().getAnalysisNumber(), entityLog.getAnalysisNumber());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testFindEntries() {
|
||||
|
||||
var latest = entityLogMongoService.findLatestAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID)
|
||||
.orElseThrow();
|
||||
var latestChangedEntries = entityLogMongoService.findAllEntityLogEntriesByAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, latest);
|
||||
assertEquals(latestChangedEntries.size(), 490);
|
||||
|
||||
var manuallyChangedEntries = entityLogMongoService.findAllEntityLogEntriesWithManualChanges(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
|
||||
assertEquals(manuallyChangedEntries.size(), 1013);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testFindLatestAnalysisNumber() {
|
||||
|
||||
assertEquals(entityLogMongoService.findLatestAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID)
|
||||
.orElse(-1), 8);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testDeleteEntityLog() {
|
||||
|
||||
entityLogMongoService.deleteEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
|
||||
assertFalse(entityLogMongoService.entityLogDocumentExists(TEST_DOSSIER_2_ID, TEST_FILE_2_ID));
|
||||
assertEquals(entityLogMongoService.findAllEntriesByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID).size(), 0);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testInsertBigEntityLogDocument() {
|
||||
|
||||
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_2_ID));
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
|
||||
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
|
||||
|
||||
entityLogMongoService.insertEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
|
||||
|
||||
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
|
||||
assertTrue(found.isPresent());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testInsertEntityLogDocument() {
|
||||
|
||||
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
|
||||
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
|
||||
|
||||
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testFindEntityLogDocumentById() {
|
||||
|
||||
Optional<EntityLog> entityLogDocumentById = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
assert (entityLogDocumentById.isPresent());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testRedactionStorageService() {
|
||||
|
||||
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
|
||||
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
|
||||
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
|
||||
|
||||
String jsonData = objectMapper.writeValueAsString(entityLogDocument);
|
||||
InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes());
|
||||
|
||||
minioInsert(jsonDataStream, jsonData.length());
|
||||
}
|
||||
|
||||
|
||||
private void minioInsert(InputStream inputStream, int length) {
|
||||
|
||||
try {
|
||||
MinioClient minioClient = MinioClient.builder()
|
||||
.endpoint("http://localhost:9000")
|
||||
.credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ")
|
||||
.build();
|
||||
|
||||
String contentType = "application/json";
|
||||
|
||||
minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID)
|
||||
.stream(inputStream, length, -1).contentType(contentType).build());
|
||||
|
||||
System.out.println("JSON file uploaded successfully!");
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, LiquibaseAutoConfiguration.class, DataSourceAutoConfiguration.class})
|
||||
@Import({MongoDbConfiguration.class, MultiTenantMongoDBFactory.class, MongoClientCache.class, MongoDataSources.class, MongoConnectionProviderImpl.class})
|
||||
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE)})
|
||||
public static class MongoTestConfiguration {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @Configuration
|
||||
* @EnableMongoRepositories
|
||||
* @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
|
||||
* @Import({LayoutParsingServiceProcessorConfiguration.class})
|
||||
* @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE)})
|
||||
* public static class MongoTestConfiguration {
|
||||
* @Bean public MongoCustomConversions mongoCustomConversions() {
|
||||
* return new MongoCustomConversions(Arrays.asList(new MongoOffsetDateTimeWriter(), new MongoOffsetDateTimeReader()
|
||||
* ));
|
||||
* }
|
||||
* <p>
|
||||
* <p>
|
||||
* }
|
||||
* <p>
|
||||
* <p>
|
||||
* public static class MongoOffsetDateTimeWriter implements Converter<OffsetDateTime, Document> {
|
||||
* <p>
|
||||
* public static final String DATE_FIELD = "dateTime";
|
||||
* public static final String OFFSET_FIELD = "offset";
|
||||
* @Override public Document convert(final OffsetDateTime offsetDateTime) {
|
||||
* final Document document = new Document();
|
||||
* document.put(DATE_FIELD, Date.from(offsetDateTime.toInstant()));
|
||||
* document.put(OFFSET_FIELD, offsetDateTime.getOffset().toString());
|
||||
* return document;
|
||||
* }
|
||||
* <p>
|
||||
* }
|
||||
* <p>
|
||||
* public static class MongoOffsetDateTimeReader implements Converter<Document, OffsetDateTime> {
|
||||
* @Override public OffsetDateTime convert(final Document document) {
|
||||
* final Date dateTime = document.getDate(MongoOffsetDateTimeWriter.DATE_FIELD);
|
||||
* final ZoneOffset offset = ZoneOffset.of(document.getString(MongoOffsetDateTimeWriter.OFFSET_FIELD));
|
||||
* return OffsetDateTime.ofInstant(dateTime.toInstant(), offset);
|
||||
* }
|
||||
* <p>
|
||||
* }
|
||||
**/
|
||||
|
||||
@Slf4j
|
||||
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
|
||||
|
||||
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
|
||||
|
||||
TestPropertyValues.of("spring.data.mongodb.auto-index-creation=true",
|
||||
"MONGODB_HOST=" + HOSTNAME,
|
||||
"MONGODB_PORT=" + PORT,
|
||||
"MONGODB_USER=" + USERNAME,
|
||||
"MONGODB_PASSWORD=" + PASSWORD,
|
||||
"storage.bucket=name=redaction",
|
||||
"storage.endpoint=http://localhost:9000",
|
||||
"storage.key=L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya",
|
||||
"storage.secret=Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ").applyTo(configurableApplicationContext.getEnvironment());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private final String NEW_ENTITY_LOG_ENTRY = """
|
||||
{
|
||||
"id": "05240998f2e657d739d59d220094702a",
|
||||
"type": "CBI_author",
|
||||
"entryType": "ENTITY",
|
||||
"state": "REMOVED",
|
||||
"value": "Amato",
|
||||
"reason": "Author found",
|
||||
"matchedRule": "CBI.0.0",
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"containingNodeId": [
|
||||
1,
|
||||
0
|
||||
],
|
||||
"closestHeadline": "",
|
||||
"section": "[1, 0]: Paragraph: Almond R.H. Almond Richard",
|
||||
"color": null,
|
||||
"positions": [
|
||||
{
|
||||
"rectangle": [
|
||||
56.8,
|
||||
639.3,
|
||||
32.59199,
|
||||
12.642
|
||||
],
|
||||
"pageNumber": 2
|
||||
}
|
||||
],
|
||||
"textBefore": "S. Amarasekare Amarasingham ",
|
||||
"textAfter": " Amato S.L Amato",
|
||||
"startOffset": 5137,
|
||||
"endOffset": 5142,
|
||||
"imageHasTransparency": false,
|
||||
"dictionaryEntry": true,
|
||||
"dossierDictionaryEntry": false,
|
||||
"excluded": false,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 1,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2024-03-13T08:44:54.811245839Z"
|
||||
},
|
||||
{
|
||||
"analysisNumber": 3,
|
||||
"type": "REMOVED",
|
||||
"dateTime": "2024-03-13T08:47:11.339124572Z"
|
||||
},
|
||||
{
|
||||
"analysisNumber": 4,
|
||||
"type": "REMOVED",
|
||||
"dateTime": "2024-03-13T08:47:19.694336707Z"
|
||||
},
|
||||
{
|
||||
"analysisNumber": 5,
|
||||
"type": "REMOVED",
|
||||
"dateTime": "2024-03-13T08:48:21.670287664Z"
|
||||
},
|
||||
{
|
||||
"analysisNumber": 6,
|
||||
"type": "REMOVED",
|
||||
"dateTime": "2024-03-13T08:58:41.082712591Z"
|
||||
},
|
||||
{
|
||||
"analysisNumber": 7,
|
||||
"type": "REMOVED",
|
||||
"dateTime": "2024-03-13T08:59:31.444615299Z"
|
||||
},
|
||||
{
|
||||
"analysisNumber": 8,
|
||||
"type": "REMOVED",
|
||||
"dateTime": "2024-03-13T09:05:22.006293189Z"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"DICTIONARY"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"numberOfComments": 0
|
||||
}
|
||||
""";
|
||||
|
||||
}
|
||||
@ -12,6 +12,15 @@ spring:
|
||||
allow-circular-references: true # FIXME
|
||||
cache:
|
||||
type: NONE
|
||||
data:
|
||||
mongodb:
|
||||
auto-index-creation: true
|
||||
# todo: multi-tenancy
|
||||
database: redaction
|
||||
host: ${MONGODB_HOST:localhost}
|
||||
port: ${MONGODB_PORT:27017}
|
||||
username: ${MONGODB_USER}
|
||||
password: ${MONGODB_PASSWORD}
|
||||
|
||||
processing.kafkastreams: false
|
||||
|
||||
@ -35,3 +44,6 @@ management:
|
||||
health.enabled: true
|
||||
endpoints.web.exposure.include: prometheus, health, metrics
|
||||
metrics.export.prometheus.enabled: true
|
||||
|
||||
|
||||
logging.level.root: debug
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,45 @@
|
||||
<!--
|
||||
#%L
|
||||
Liquibase MongoDB Extension
|
||||
%%
|
||||
Copyright (C) 2019 Mastercard
|
||||
%%
|
||||
Licensed under the Apache License, Version 2.0 (the "License").
|
||||
You may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
#L%
|
||||
-->
|
||||
<databaseChangeLog
|
||||
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:mongodb="http://www.liquibase.org/xml/ns/mongodb"
|
||||
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.6.xsd
|
||||
http://www.liquibase.org/xml/ns/mongodb http://www.liquibase.org/xml/ns/mongodb/liquibase-mongodb-latest.xsd">
|
||||
|
||||
<changeSet id="1" author="maverick">
|
||||
<mongodb:runCommand>
|
||||
<mongodb:command>
|
||||
{
|
||||
update: "entity-logs",
|
||||
updates: [
|
||||
{
|
||||
q: {},
|
||||
u: { $rename: { "dId": "dossierId" } },
|
||||
multi: true
|
||||
}
|
||||
]
|
||||
}
|
||||
</mongodb:command>
|
||||
|
||||
</mongodb:runCommand>
|
||||
</changeSet>
|
||||
|
||||
</databaseChangeLog>
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
x
Reference in New Issue
Block a user