diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java index 0cdb2ca51..0f58f9dad 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java @@ -36,7 +36,7 @@ public class AdminInterfaceController { public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); + fileManagementStorageService.deleteEntityLog(dossierId, fileId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION); @@ -141,7 +141,7 @@ public class AdminInterfaceController { var fileId = file.getId(); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); + fileManagementStorageService.deleteEntityLog(dossierId, fileId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); diff --git a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts index 1fb9d4bcf..0e277da9a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-processor-v1/build.gradle.kts @@ -34,8 +34,8 @@ dependencies { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } api("com.knecon.fforesight:jobs-commons:0.10.0") - api("com.knecon.fforesight:database-tenant-commons:0.21.0") - api("com.knecon.fforesight:keycloak-commons:0.25.0") + api("com.knecon.fforesight:database-tenant-commons:maverick-mongo") + api("com.knecon.fforesight:keycloak-commons:maverick-mongo") api("com.knecon.fforesight:tracing-commons:0.5.0") api("com.knecon.fforesight:swagger-commons:0.5.0") api("com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:0.4.0") @@ -43,6 +43,7 @@ dependencies { api("org.springframework.boot:spring-boot-starter-mail:${springBootStarterVersion}") api("org.springframework.boot:spring-boot-starter-data-jpa:${springBootStarterVersion}") api("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}") + api("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}") api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}") api("org.springframework.boot:spring-boot-starter-web:${springBootStarterVersion}") api("com.iqser.red.commons:spring-commons:2.1.0") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogDocument.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogDocument.java new file mode 100644 index 000000000..48ebf6651 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogDocument.java @@ -0,0 +1,70 @@ +package com.iqser.red.service.persistence.management.v1.processor.document; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@Document(collection = "entity-logs") +public class EntityLogDocument { + + @Id + @EqualsAndHashCode.Include + private String id; + + private String dossierId; + private String fileId; + + private long analysisVersion; + + //@Indexed(direction = IndexDirection.DESCENDING) + private int analysisNumber; + + @DBRef + private List entityLogEntryDocument = new ArrayList<>(); + + private List legalBasis = new ArrayList<>(); + + private long dictionaryVersion = -1; + private long dossierDictionaryVersion = -1; + + private long rulesVersion = -1; + private long legalBasisVersion = -1; + + + public EntityLogDocument(String dossierId, String fileId, EntityLog entityLog) { + + MagicConverter.copyAllFields(entityLog, this); + this.id = getDocumentId(dossierId, fileId); + this.dossierId = dossierId; + this.fileId = fileId; + this.entityLogEntryDocument = new ArrayList<>(entityLog.getEntityLogEntry() + .stream() + .map(entityLogEntry -> new EntityLogEntryDocument(this.id, entityLogEntry)).toList()); + + } + + + public static String getDocumentId(String dossierId, String fileId) { + + return dossierId + "/" + fileId; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogEntryDocument.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogEntryDocument.java new file mode 100644 index 000000000..aa20ebd04 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogEntryDocument.java @@ -0,0 +1,94 @@ +package com.iqser.red.service.persistence.management.v1.processor.document; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +@FieldDefaults(level = AccessLevel.PRIVATE) +@Document(collection = "entity-log-entries") +//@CompoundIndexes({ + //@CompoundIndex(name = "changes_analysisNumber", def = "{ 'changes.analysisNumber': 1 }") +// }) +public class EntityLogEntryDocument { + + @Id + @EqualsAndHashCode.Include + String id; + String entryId; + String entityLogId; + String type; + EntryType entryType; + EntryState state; + String value; + String reason; + String matchedRule; + String legalBasis; + + List containingNodeId; + String closestHeadline; + String section; + + List positions = new ArrayList<>(); + + String textBefore; + String textAfter; + + int startOffset; + int endOffset; + + boolean imageHasTransparency; + + boolean dictionaryEntry; + boolean dossierDictionaryEntry; + + boolean excluded; + + List changes = new ArrayList<>(); + + List manualChanges = new ArrayList<>(); + + Set engines = new HashSet<>(); + + Set reference = new HashSet<>(); + + Set importedRedactionIntersections = new HashSet<>(); + + int numberOfComments; + + + public EntityLogEntryDocument(String entityLogId, EntityLogEntry entityLogEntry) { + + MagicConverter.copyAllFields(entityLogEntry, this); + this.id = entityLogId + "/" + entityLogEntry.getId(); + this.entryId = entityLogEntry.getId(); + this.entityLogId = entityLogId; + + + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/exception/EntityLogDocumentNotFoundException.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/exception/EntityLogDocumentNotFoundException.java new file mode 100644 index 000000000..fe33d0df9 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/exception/EntityLogDocumentNotFoundException.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.persistence.management.v1.processor.exception; + +public class EntityLogDocumentNotFoundException extends RuntimeException { + + public EntityLogDocumentNotFoundException(String errorMessage) { + + super(errorMessage); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java index 3606f9a74..46444ec4b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java @@ -45,6 +45,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE; +import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType.ENTITY_LOG; import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; @Slf4j @@ -200,9 +201,10 @@ public class SaasMigrationService implements TenantSyncService { } + //todo: 8702 private boolean entityLogMigrationFilesExist(String dossierId, String fileId) { - return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists( + return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, ENTITY_LOG)) && storageService.objectExists( TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS)); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeRenameMigration15.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeRenameMigration15.java index c42fdcfe3..8494b32e2 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeRenameMigration15.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeRenameMigration15.java @@ -69,7 +69,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration { continue; } - if (!fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.ENTITY_LOG)) { + if (!fileManagementStorageService.entityLogExists(file.getDossierId(), file.getId())) { continue; } @@ -92,7 +92,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration { } } } - fileManagementStorageService.storeJSONObject(file.getDossierId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(file.getDossierId(), file.getId(), entityLog); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java index b199fb4ae..8036fb11f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService; import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; @@ -36,6 +37,8 @@ public class FileManagementStorageService { private final StorageService storageService; + private final EntityLogMongoService entityLogMongoService; + @SneakyThrows public byte[] getStoredObjectBytes(String dossierId, String fileId, FileType fileType) { @@ -99,14 +102,20 @@ public class FileManagementStorageService { public EntityLog getEntityLog(String dossierId, String fileId) { - try { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class); - } catch (StorageObjectDoesNotExist e) { - log.debug("EntityLog does not exist"); - throw new NotFoundException(String.format("EntityLog does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId)); - } catch (StorageException e) { - throw new InternalServerErrorException(e.getMessage()); - } + return entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId) + .orElseThrow(() -> new NotFoundException(String.format("EntityLog does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId))); + + } + + @SneakyThrows + public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) { + + entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog); + } + + public boolean entityLogExists(String dossierId, String fileId) { + + return entityLogMongoService.entityLogDocumentExists(dossierId, fileId); } @@ -161,6 +170,11 @@ public class FileManagementStorageService { storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType)); } + public void deleteEntityLog(String dossierId, String fileId) { + + entityLogMongoService.deleteEntityLog(dossierId, fileId); + } + public void deleteObject(String storageId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 22273f6a8..4ad2d6cf3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -690,7 +690,7 @@ public class FileStatusService { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); + fileManagementStorageService.deleteEntityLog(dossierId, fileId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES); @@ -719,7 +719,7 @@ public class FileStatusService { OffsetDateTime now = OffsetDateTime.now(); // remove everything related to redaction fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); - fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG); + fileManagementStorageService.deleteEntityLog(dossierId, fileId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); // wipe comments diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/EntityLogMongoService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/EntityLogMongoService.java new file mode 100644 index 000000000..87ebd379d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/EntityLogMongoService.java @@ -0,0 +1,238 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.mongo; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument; +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument; +import com.iqser.red.service.persistence.management.v1.processor.exception.EntityLogDocumentNotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogDocumentRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogEntryDocumentRepository; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; + +@Service +public class EntityLogMongoService { + + private final EntityLogDocumentRepository entityLogDocumentRepository; + private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository; + + + public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository, EntityLogEntryDocumentRepository entityLogEntryDocumentRepository) { + + this.entityLogDocumentRepository = entityLogDocumentRepository; + this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository; + } + + + public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) { + + EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog)); + + entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry() + .stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry)) + .toList()); + } + + + // this does everything : insert when not found and update if found + // todo: remove and replace when services use insert,update,delete correctly + public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) { + + Optional optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId)); + if (optionalEntityLogDocument.isEmpty()) { + // throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId)); + insertEntityLog(dossierId, fileId, entityLog); + return; + } + + EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get(); + List oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument(); + + EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog); + List newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument(); + + List toUpdate = new ArrayList<>(oldEntityLogEntryDocuments); + toUpdate.retainAll(newEntityLogEntryDocuments); + + List toRemove = new ArrayList<>(oldEntityLogEntryDocuments); + toRemove.removeAll(toUpdate); + + List toInsert = new ArrayList<>(newEntityLogEntryDocuments); + toInsert.removeAll(toUpdate); + + entityLogEntryDocumentRepository.saveAll(toUpdate); + entityLogEntryDocumentRepository.deleteAll(toRemove); + entityLogEntryDocumentRepository.insert(toInsert); + + entityLogDocumentRepository.save(newEntityLogDocument); + } + + + public void deleteEntityLog(String dossierId, String fileId) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + entityLogDocumentRepository.deleteById(entityLogId); + + entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId); + } + + + public void insertEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId); + + List entityLogEntryDocuments = entityLogEntries.stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry)) + .toList(); + + entityLogDocument.getEntityLogEntryDocument().addAll(entityLogEntryDocuments); + + entityLogEntryDocumentRepository.insert(entityLogEntryDocuments); + entityLogDocumentRepository.save(entityLogDocument); + } + + + public void updateEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry)) + .toList()); + } + + public void deleteEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { + + String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId); + + EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId); + + List entityLogEntryDocuments = entityLogEntries.stream() + .map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry)) + .toList(); + + entityLogDocument.getEntityLogEntryDocument().removeAll(entityLogEntryDocuments); + + entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments); + entityLogDocumentRepository.save(entityLogDocument); + } + + + private EntityLogDocument getEntityLogDocument(String entityLogId) { + + Optional optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId); + + if (optionalEntityLogDocument.isEmpty()) { + throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId)); + } + + return optionalEntityLogDocument.get(); + } + + + public Optional findEntityLogByDossierIdAndFileId(String dossierId, String fileId) { + + return entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId)) + .map(EntityLogMongoService::fromDocument); + } + + + public boolean entityLogDocumentExists(String dossierId, String fileId) { + + return entityLogDocumentRepository.existsById(EntityLogDocument.getDocumentId(dossierId, fileId)); + } + + + public Optional findLatestAnalysisNumber(String dossierId, String fileId) { + + return entityLogDocumentRepository.findAnalysisNumberById(EntityLogDocument.getDocumentId(dossierId, fileId)) + .map(EntityLogDocument::getAnalysisNumber); + } + + + public List findAllEntityLogEntriesWithManualChanges(String dossierId, String fileId) { + + return entityLogEntryDocumentRepository.findByEntityLogIdAndManualChangesNotEmpty(EntityLogDocument.getDocumentId(dossierId, fileId)) + .stream() + .map(EntityLogMongoService::fromDocument) + .toList(); + } + + + public List findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) { + + return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(EntityLogDocument.getDocumentId(dossierId, fileId), analysisNumber) + .stream() + .map(EntityLogMongoService::fromDocument) + .toList(); + } + + + public List findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) { + + return entityLogEntryDocumentRepository.findByEntityLogId(EntityLogDocument.getDocumentId(dossierId, fileId)) + .stream() + .map(EntityLogMongoService::fromDocument) + .toList(); + } + + + private static EntityLog fromDocument(EntityLogDocument entityLogDocument) { + + EntityLog entityLog = new EntityLog(); + entityLog.setAnalysisVersion(entityLogDocument.getAnalysisVersion()); + entityLog.setAnalysisNumber(entityLogDocument.getAnalysisNumber()); + entityLog.setEntityLogEntry(entityLogDocument.getEntityLogEntryDocument() + .stream() + .map(EntityLogMongoService::fromDocument) + .collect(Collectors.toList())); + entityLog.setLegalBasis(entityLogDocument.getLegalBasis()); + entityLog.setDictionaryVersion(entityLogDocument.getDictionaryVersion()); + entityLog.setDossierDictionaryVersion(entityLogDocument.getDossierDictionaryVersion()); + entityLog.setRulesVersion(entityLogDocument.getRulesVersion()); + entityLog.setLegalBasisVersion(entityLogDocument.getLegalBasisVersion()); + return entityLog; + } + + + private static EntityLogEntry fromDocument(EntityLogEntryDocument entityLogEntryDocument) { + + EntityLogEntry entityLogEntry = new EntityLogEntry(); + entityLogEntry.setId(entityLogEntryDocument.getEntryId()); + entityLogEntry.setState(entityLogEntryDocument.getState()); + entityLogEntry.setValue(entityLogEntryDocument.getValue()); + entityLogEntry.setReason(entityLogEntryDocument.getReason()); + entityLogEntry.setMatchedRule(entityLogEntryDocument.getMatchedRule()); + entityLogEntry.setLegalBasis(entityLogEntryDocument.getLegalBasis()); + entityLogEntry.setContainingNodeId(new ArrayList<>(entityLogEntryDocument.getContainingNodeId())); + entityLogEntry.setClosestHeadline(entityLogEntryDocument.getClosestHeadline()); + entityLogEntry.setSection(entityLogEntryDocument.getSection()); + entityLogEntry.setPositions(new ArrayList<>(entityLogEntryDocument.getPositions())); + entityLogEntry.setTextBefore(entityLogEntryDocument.getTextBefore()); + entityLogEntry.setTextAfter(entityLogEntryDocument.getTextAfter()); + entityLogEntry.setStartOffset(entityLogEntryDocument.getStartOffset()); + entityLogEntry.setEndOffset(entityLogEntryDocument.getEndOffset()); + entityLogEntry.setImageHasTransparency(entityLogEntryDocument.isImageHasTransparency()); + entityLogEntry.setDictionaryEntry(entityLogEntryDocument.isDictionaryEntry()); + entityLogEntry.setDossierDictionaryEntry(entityLogEntryDocument.isDossierDictionaryEntry()); + entityLogEntry.setExcluded(entityLogEntryDocument.isExcluded()); + entityLogEntry.setChanges(new ArrayList<>(entityLogEntryDocument.getChanges())); + entityLogEntry.setManualChanges(new ArrayList<>(entityLogEntryDocument.getManualChanges())); + entityLogEntry.setEngines(new HashSet<>(entityLogEntryDocument.getEngines())); + entityLogEntry.setReference(new HashSet<>(entityLogEntryDocument.getReference())); + entityLogEntry.setImportedRedactionIntersections(new HashSet<>(entityLogEntryDocument.getImportedRedactionIntersections())); + entityLogEntry.setNumberOfComments(entityLogEntryDocument.getNumberOfComments()); + return entityLogEntry; + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogDocumentRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogDocumentRepository.java new file mode 100644 index 000000000..d7aa23952 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogDocumentRepository.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository; + +import java.util.Optional; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument; + +@Repository +public interface EntityLogDocumentRepository extends MongoRepository { + + @Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }") + Optional findAnalysisNumberById(@Param("_id") String id); +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogEntryDocumentRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogEntryDocumentRepository.java new file mode 100644 index 000000000..c8e4f2015 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogEntryDocumentRepository.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository; + +import java.util.List; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.stereotype.Repository; + +import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument; + +@Repository +public interface EntityLogEntryDocumentRepository extends MongoRepository { + + @Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }") + List findByEntityLogIdAndManualChangesNotEmpty(String entityLogId); + + @Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }") + List findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber); + + @Query("{ 'entityLogId' : ?0}") + List findByEntityLogId(String entityLogId); + + @Query(value = "{ 'entityLogId' : ?0}", delete = true) + void deleteByEntityLogId(String entityLogId); +} diff --git a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts index 521247b80..56535f7ff 100644 --- a/persistence-service-v1/persistence-service-server-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-server-v1/build.gradle.kts @@ -23,6 +23,7 @@ dependencies { api(project(":persistence-service-external-api-impl-v2")) api(project(":persistence-service-internal-api-impl-v1")) api("com.iqser.red.commons:storage-commons:2.45.0") + api("com.knecon.fforesight:mongo-database-commons:maverick-mongo") api("junit:junit:4.13.2") api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0") api("net.logstash.logback:logstash-logback-encoder:7.4") diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java index 50f039237..a90522e03 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java @@ -21,6 +21,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.http.server.observation.DefaultServerRequestObservationConvention; import org.springframework.http.server.observation.ServerRequestObservationContext; import org.springframework.http.server.observation.ServerRequestObservationConvention; @@ -43,6 +44,7 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.knecon.fforesight.databasetenantcommons.DatabaseTenantCommonsAutoConfiguration; import com.knecon.fforesight.jobscommons.JobsAutoConfiguration; import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration; +import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration; import com.knecon.fforesight.swaggercommons.SpringDocAutoConfiguration; import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration; import com.knecon.fforesight.tenantcommons.MultiTenancyMessagingConfiguration; @@ -62,7 +64,8 @@ import lombok.extern.slf4j.Slf4j; @EnableScheduling @EnableCaching @EnableConfigurationProperties({FileManagementServiceSettings.class}) -@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class}) +@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence") +@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class}) @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, CassandraAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class}) @Import({PersistenceServiceExternalApiConfigurationV2.class, PersistenceServiceExternalApiConfiguration.class, PersistenceServiceInternalApiConfiguration.class, PersistenceServiceExternalApiCacheConfiguration.class, MultiTenancyWebConfiguration.class, PersistenceServiceProcessorConfiguration.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class}) public class Application implements ApplicationContextAware { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml index 0af757fe1..6960236c6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml @@ -37,6 +37,15 @@ spring: batch_size: 1000 order_inserts: true order_updates: true + data: + mongodb: + auto-index-creation: true + # todo: multi-tenancy + database: redaction + host: ${MONGODB_HOST:localhost} + port: 27017 + username: ${MONGODB_USER} + password: ${MONGODB_PASSWORD} cache: type: redis mvc: diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index e856e1b16..d702fb1c6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -64,9 +64,8 @@ public class FileTesterAndProvider { assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1); - fileManagementStorageService.storeJSONObject(dossier.getId(), + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getFileId(), - FileType.ENTITY_LOG, new EntityLog(1, 1, List.of(EntityLogEntry.builder() diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java index 7b6107283..7672880cc 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java @@ -139,7 +139,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyBoolean())).thenReturn(entityLog); @@ -333,7 +333,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); @@ -393,7 +393,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); @@ -448,7 +448,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); @@ -543,7 +543,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1); + fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1); var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(eq(file1.getDossierId()), eq(file1.getFileId()), any(), anyBoolean())).thenReturn(entityLog1); @@ -562,7 +562,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2); + fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2); var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2); @@ -708,7 +708,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1); + fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1); var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(eq(file1.getDossierId()), eq(file1.getFileId()), any(), anyBoolean())).thenReturn(entityLog1); @@ -727,7 +727,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2); + fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2); var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2); @@ -877,7 +877,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1); + fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1); var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(file1.getDossierId(), file1.getFileId())).thenReturn(entityLog1); @@ -896,7 +896,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2); + fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2); var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2); @@ -1043,7 +1043,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1); + fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1); var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(file1.getDossierId(), file1.getFileId())).thenReturn(entityLog1); @@ -1062,7 +1062,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2); + fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2); var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build(); when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2); @@ -1186,7 +1186,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); @@ -1270,7 +1270,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); @@ -1449,7 +1449,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); @@ -1648,7 +1648,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); @@ -1754,7 +1754,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(entityLog); @@ -1864,7 +1864,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(entityLog); @@ -1948,8 +1948,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest { 0, 0, 0); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); - fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); + fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog); when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog); var recatModel = RecategorizationRequestModel.builder() diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index 76eb9a805..6a2c43035 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -1,42 +1,17 @@ package com.iqser.red.service.peristence.v1.server.integration.utils; -import com.iqser.red.commons.jackson.ObjectMapperFactory; -import com.iqser.red.service.peristence.v1.server.Application; -import com.iqser.red.service.peristence.v1.server.integration.client.ApplicationConfigClient; -import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; -import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService; -import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient; -import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.RedactionClient; -import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient; -import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity; -import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles; -import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService; -import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService; -import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository; -import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig; -import com.iqser.red.service.redaction.v1.model.DroolsValidation; -import com.iqser.red.storage.commons.service.StorageService; -import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; -import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener; -import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent; -import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; -import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService; -import com.knecon.fforesight.tenantcommons.TenantContext; -import com.knecon.fforesight.tenantcommons.TenantsClient; -import com.knecon.fforesight.tenantcommons.model.*; +import static org.mockito.Mockito.when; -import io.micrometer.prometheus.PrometheusMeterRegistry; -import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.assertj.core.util.Lists; +import org.bson.BsonArray; +import org.bson.BsonDocument; +import org.bson.BsonString; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; @@ -53,7 +28,11 @@ import org.springframework.boot.test.util.TestPropertyValues; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.*; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.StatementCallback; import org.springframework.jdbc.datasource.SingleConnectionDataSource; @@ -69,13 +48,78 @@ import org.springframework.security.web.SecurityFilterChain; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.service.peristence.v1.server.Application; +import com.iqser.red.service.peristence.v1.server.integration.client.ApplicationConfigClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; +import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService; +import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient; +import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.RedactionClient; +import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient; +import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity; +import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles; +import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService; +import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService; +import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogDocumentRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogEntryDocumentRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeConfigRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierStatusRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.IndexInformationRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationPreferencesRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RuleSetRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ForceRedactionRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.LegalBasisChangeRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ManualRedactionRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RecategorizationRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig; +import com.iqser.red.service.redaction.v1.model.DroolsValidation; +import com.iqser.red.storage.commons.service.StorageService; +import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener; +import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent; +import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter; +import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService; +import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.tenantcommons.TenantsClient; +import com.knecon.fforesight.tenantcommons.model.AuthDetails; +import com.knecon.fforesight.tenantcommons.model.DatabaseConnection; +import com.knecon.fforesight.tenantcommons.model.MongoDBConnection; +import com.knecon.fforesight.tenantcommons.model.S3StorageConnection; +import com.knecon.fforesight.tenantcommons.model.SearchConnection; +import com.knecon.fforesight.tenantcommons.model.TenantResponse; +import com.mongodb.MongoCommandException; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; -import static org.mockito.Mockito.when; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import lombok.extern.slf4j.Slf4j; @Slf4j @ExtendWith(SpringExtension.class) @@ -100,6 +144,12 @@ public abstract class AbstractPersistenceServerServiceTest { @Autowired protected StorageService storageService; @Autowired + protected EntityLogMongoService entityLogMongoService; + @Autowired + protected EntityLogDocumentRepository entityLogDocumentRepository; + @Autowired + protected EntityLogEntryDocumentRepository entityLogEntryDocumentRepository; + @Autowired protected FileManagementStorageService fileManagementStorageService; @Autowired protected DossierTemplateRepository dossierTemplateRepository; @@ -268,6 +318,7 @@ public abstract class AbstractPersistenceServerServiceTest { private void createDefaultTenant() { var postgreSQLContainerMaster = SpringPostgreSQLTestContainer.getInstance().withDatabaseName("integration-tests-db-master").withUsername("sa").withPassword("sa"); + var mongoDbContainer = MongoDBTestContainer.getInstance(); var port = postgreSQLContainerMaster.getJdbcUrl().substring(0, postgreSQLContainerMaster.getJdbcUrl().lastIndexOf('/')).split(":")[3]; @@ -296,6 +347,7 @@ public abstract class AbstractPersistenceServerServiceTest { .numberOfShards("1") .numberOfReplicas("5") .build()); + redactionTenant.setS3StorageConnection(S3StorageConnection.builder() .key("key") .secret("secret") @@ -304,6 +356,15 @@ public abstract class AbstractPersistenceServerServiceTest { .region("eu") .endpoint("endpoint") .build()); + + redactionTenant.setMongoDBConnection(MongoDBConnection.builder() + .host(mongoDbContainer.getHost()) + .port(String.valueOf(mongoDbContainer.getFirstMappedPort())) + .username(MongoDBTestContainer.MONGO_USERNAME) + .password(encryptionDecryptionService.encrypt(MongoDBTestContainer.MONGO_PASSWORD)) + .database(MongoDBTestContainer.MONGO_DATABASE) + .build()); + when(tenantsClient.getTenant("redaction")).thenReturn(redactionTenant); when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant)); @@ -332,6 +393,8 @@ public abstract class AbstractPersistenceServerServiceTest { } + + @AfterEach public void cleanupStorage() { @@ -385,6 +448,9 @@ public abstract class AbstractPersistenceServerServiceTest { indexInformationRepository.deleteAll(); applicationConfigRepository.deleteAll(); + entityLogEntryDocumentRepository.deleteAll(); + entityLogDocumentRepository.deleteAll(); + }); } @@ -401,10 +467,18 @@ public abstract class AbstractPersistenceServerServiceTest { var redisContainer = RedisTestContainer.getInstance(); redisContainer.start(); - log.info("Hosts are - Redis: {}, Postgres: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost()); + var mongoInstance = MongoDBTestContainer.getInstance(); + mongoInstance.start(); + createMongoDBDatabase(mongoInstance); + + log.info("Hosts are - Redis: {}, Postgres: {}, MongoDB: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost(), mongoInstance.getHost()); TestPropertyValues.of("REDIS_PORT=" + redisContainer.getFirstMappedPort(), "REDIS_HOST=" + redisContainer.getHost(), + "MONGODB_HOST=" + mongoInstance.getHost(), + "MONGODB_PORT=" + mongoInstance.getFirstMappedPort(), + "MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME, + "MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD, "fforesight.jobs.enabled=false", "fforesight.keycloak.enabled=false").applyTo(configurableApplicationContext.getEnvironment()); @@ -412,11 +486,40 @@ public abstract class AbstractPersistenceServerServiceTest { } + + private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) { + + try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/", + MongoDBTestContainer.MONGO_USERNAME, + MongoDBTestContainer.MONGO_PASSWORD, + mongoDBTestContainer.getHost(), + mongoDBTestContainer.getFirstMappedPort()))) { + MongoDatabase database = mongoClient.getDatabase(MongoDBTestContainer.MONGO_DATABASE); + BsonDocument createUserCommand = new BsonDocument(); + createUserCommand.append("createUser", new BsonString(MongoDBTestContainer.MONGO_USERNAME)); + createUserCommand.append("pwd", new BsonString(MongoDBTestContainer.MONGO_PASSWORD)); + BsonArray roles = new BsonArray(); + roles.add(new BsonString("readWrite")); + createUserCommand.append("roles", roles); + + try { + database.runCommand(createUserCommand); + } catch (MongoCommandException mongoCommandException) { + // ignore user already exists + if(mongoCommandException.getErrorCode() != 51003) { + throw mongoCommandException; + } + + } + + } + } + @Configuration @EnableWebSecurity @EnableMethodSecurity @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) - @ComponentScan("com.iqser.red.service.persistence") + @ComponentScan(basePackages = {"com.iqser.red.service.persistence" }) public static class TestConfiguration { @Bean diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/MongoDBTestContainer.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/MongoDBTestContainer.java new file mode 100644 index 000000000..6db3c46ec --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/MongoDBTestContainer.java @@ -0,0 +1,35 @@ +package com.iqser.red.service.peristence.v1.server.integration.utils; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +public final class MongoDBTestContainer extends GenericContainer { + + private static final String IMAGE_VERSION = "mongo:7.0.2"; + public static final Integer MONGO_PORT = 27017; + public static final String MONGO_DATABASE = "redaction"; + public static final String MONGO_PASSWORD = "mongopassword"; + public static final String MONGO_USERNAME = "mongousername"; + private static MongoDBTestContainer mongoDB; + + + private MongoDBTestContainer() { + + super(DockerImageName.parse(IMAGE_VERSION)); + + } + + + public static MongoDBTestContainer getInstance() { + + if (mongoDB == null) { + mongoDB = new MongoDBTestContainer().withEnv("MONGO_INITDB_ROOT_USERNAME", MONGO_USERNAME) + .withEnv("MONGO_INITDB_ROOT_PASSWORD", MONGO_PASSWORD) + .withEnv("MONGO_INITDB_DATABASE", MONGO_DATABASE) + .withExposedPorts(MONGO_PORT); + + } + return mongoDB; + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml index 968397e81..7f5c1f6a7 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml @@ -19,7 +19,15 @@ spring: order_inserts: true order_updates: true open-in-view: true - + data: + mongodb: + auto-index-creation: true + # todo: multi-tenancy + database: redaction + host: ${MONGODB_HOST:localhost} + port: 27017 + username: ${MONGODB_USER} + password: ${MONGODB_PASSWORD} rabbitmq: host: ${RABBITMQ_HOST:localhost} port: ${RABBITMQ_PORT:5672} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts index a10ac954d..b2dfa2a93 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts +++ b/persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts @@ -3,6 +3,8 @@ plugins { id("io.freefair.lombok") version "8.4" } +val springBootStarterVersion = "3.1.5" + dependencies { api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0") api("com.google.guava:guava:31.1-jre")