Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* refactoring

RED-8702: Explore document databases to store entityLog
* refactoring

RED-8702: Explore document databases to store entityLog
* fix for AbstractPersistenceServerServiceTest to include mongodb container

RED-8702: Explore document databases to store entityLog
* first fix for failing tests

RED-8702: Explore document databases to store entityLog

RED-8702: Explore document databases to store entityLog
* added mongodb documents, repositories and EntityLogMongoService
* refactored places where entity logs were stored and loaded to use new logic

RED-8702

RED-8702
This commit is contained in:
maverickstuder 2024-03-13 15:26:35 +01:00
parent 9110e68266
commit 49932a9074
21 changed files with 715 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/internal/AdminInterfaceController.java
View File

@ -36,7 +36,7 @@ public class AdminInterfaceController {
public void resetFile(@RequestParam("dossierId") String dossierId, @RequestParam("fileId") String fileId) {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_POSITION);
@ -141,7 +141,7 @@ public class AdminInterfaceController {
var fileId = file.getId();
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_TEXT);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);

5
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -34,8 +34,8 @@ dependencies {
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.knecon.fforesight:database-tenant-commons:0.21.0")
api("com.knecon.fforesight:keycloak-commons:0.25.0")
api("com.knecon.fforesight:database-tenant-commons:maverick-mongo")
api("com.knecon.fforesight:keycloak-commons:maverick-mongo")
api("com.knecon.fforesight:tracing-commons:0.5.0")
api("com.knecon.fforesight:swagger-commons:0.5.0")
api("com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:0.4.0")
@ -43,6 +43,7 @@ dependencies {
api("org.springframework.boot:spring-boot-starter-mail:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-jpa:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-web:${springBootStarterVersion}")
api("com.iqser.red.commons:spring-commons:2.1.0")

70
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogDocument.java Normal file
View File

@ -0,0 +1,70 @@
package com.iqser.red.service.persistence.management.v1.processor.document;
import java.util.ArrayList;
import java.util.List;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@Document(collection = "entity-logs")
public class EntityLogDocument {
@Id
@EqualsAndHashCode.Include
private String id;
private String dossierId;
private String fileId;
private long analysisVersion;
//@Indexed(direction = IndexDirection.DESCENDING)
private int analysisNumber;
@DBRef
private List<EntityLogEntryDocument> entityLogEntryDocument = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
public EntityLogDocument(String dossierId, String fileId, EntityLog entityLog) {
MagicConverter.copyAllFields(entityLog, this);
this.id = getDocumentId(dossierId, fileId);
this.dossierId = dossierId;
this.fileId = fileId;
this.entityLogEntryDocument = new ArrayList<>(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(this.id, entityLogEntry)).toList());
}
public static String getDocumentId(String dossierId, String fileId) {
return dossierId + "/" + fileId;
}
}

94
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/document/EntityLogEntryDocument.java Normal file
View File

@ -0,0 +1,94 @@
package com.iqser.red.service.persistence.management.v1.processor.document;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
@Document(collection = "entity-log-entries")
//@CompoundIndexes({
//@CompoundIndex(name = "changes_analysisNumber", def = "{ 'changes.analysisNumber': 1 }")
// })
public class EntityLogEntryDocument {
@Id
@EqualsAndHashCode.Include
String id;
String entryId;
String entityLogId;
String type;
EntryType entryType;
EntryState state;
String value;
String reason;
String matchedRule;
String legalBasis;
List<Integer> containingNodeId;
String closestHeadline;
String section;
List<Position> positions = new ArrayList<>();
String textBefore;
String textAfter;
int startOffset;
int endOffset;
boolean imageHasTransparency;
boolean dictionaryEntry;
boolean dossierDictionaryEntry;
boolean excluded;
List<Change> changes = new ArrayList<>();
List<ManualChange> manualChanges = new ArrayList<>();
Set<Engine> engines = new HashSet<>();
Set<String> reference = new HashSet<>();
Set<String> importedRedactionIntersections = new HashSet<>();
int numberOfComments;
public EntityLogEntryDocument(String entityLogId, EntityLogEntry entityLogEntry) {
MagicConverter.copyAllFields(entityLogEntry, this);
this.id = entityLogId + "/" + entityLogEntry.getId();
this.entryId = entityLogEntry.getId();
this.entityLogId = entityLogId;
}
}

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/exception/EntityLogDocumentNotFoundException.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.exception;
public class EntityLogDocumentNotFoundException extends RuntimeException {
public EntityLogDocumentNotFoundException(String errorMessage) {
super(errorMessage);
}
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/SaasMigrationService.java
View File

@ -45,6 +45,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration.MIGRATION_QUEUE;
import static com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType.ENTITY_LOG;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
@Slf4j
@ -200,9 +201,10 @@ public class SaasMigrationService implements TenantSyncService {
}
//todo: 8702
private boolean entityLogMigrationFilesExist(String dossierId, String fileId) {
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG)) && storageService.objectExists(
return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, ENTITY_LOG)) && storageService.objectExists(
TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.MIGRATED_IDS));
}

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/ManualRedactionTypeRenameMigration15.java
View File

@ -69,7 +69,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration {
continue;
}
if (!fileManagementStorageService.objectExists(file.getDossierId(), file.getId(), FileType.ENTITY_LOG)) {
if (!fileManagementStorageService.entityLogExists(file.getDossierId(), file.getId())) {
continue;
}
@ -92,7 +92,7 @@ public class ManualRedactionTypeRenameMigration15 extends Migration {
}
}
}
fileManagementStorageService.storeJSONObject(file.getDossierId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(file.getDossierId(), file.getId(), entityLog);
}
}

30
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java
View File

@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -36,6 +37,8 @@ public class FileManagementStorageService {
private final StorageService storageService;
private final EntityLogMongoService entityLogMongoService;
@SneakyThrows
public byte[] getStoredObjectBytes(String dossierId, String fileId, FileType fileType) {
@ -99,14 +102,20 @@ public class FileManagementStorageService {
public EntityLog getEntityLog(String dossierId, String fileId) {
try {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
} catch (StorageObjectDoesNotExist e) {
log.debug("EntityLog does not exist");
throw new NotFoundException(String.format("EntityLog does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId));
} catch (StorageException e) {
throw new InternalServerErrorException(e.getMessage());
}
return entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId)
.orElseThrow(() -> new NotFoundException(String.format("EntityLog does not exist for Dossier ID \"%s\" and File ID \"%s\"!", dossierId, fileId)));
}
@SneakyThrows
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog);
}
public boolean entityLogExists(String dossierId, String fileId) {
return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
}
@ -161,6 +170,11 @@ public class FileManagementStorageService {
storageService.deleteObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType));
}
public void deleteEntityLog(String dossierId, String fileId) {
entityLogMongoService.deleteEntityLog(dossierId, fileId);
}
public void deleteObject(String storageId) {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -690,7 +690,7 @@ public class FileStatusService {
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.DOCUMENT_PAGES);
@ -719,7 +719,7 @@ public class FileStatusService {
OffsetDateTime now = OffsetDateTime.now();
// remove everything related to redaction
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ENTITY_LOG);
fileManagementStorageService.deleteEntityLog(dossierId, fileId);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
// wipe comments

238
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/EntityLogMongoService.java Normal file
View File

@ -0,0 +1,238 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.management.v1.processor.exception.EntityLogDocumentNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@Service
public class EntityLogMongoService {
private final EntityLogDocumentRepository entityLogDocumentRepository;
private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository, EntityLogEntryDocumentRepository entityLogEntryDocumentRepository) {
this.entityLogDocumentRepository = entityLogDocumentRepository;
this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
}
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog));
entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
.toList());
}
// this does everything : insert when not found and update if found
// todo: remove and replace when services use insert,update,delete correctly
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId));
if (optionalEntityLogDocument.isEmpty()) {
// throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId));
insertEntityLog(dossierId, fileId, entityLog);
return;
}
EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get();
List<EntityLogEntryDocument> oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument();
EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog);
List<EntityLogEntryDocument> newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument();
List<EntityLogEntryDocument> toUpdate = new ArrayList<>(oldEntityLogEntryDocuments);
toUpdate.retainAll(newEntityLogEntryDocuments);
List<EntityLogEntryDocument> toRemove = new ArrayList<>(oldEntityLogEntryDocuments);
toRemove.removeAll(toUpdate);
List<EntityLogEntryDocument> toInsert = new ArrayList<>(newEntityLogEntryDocuments);
toInsert.removeAll(toUpdate);
entityLogEntryDocumentRepository.saveAll(toUpdate);
entityLogEntryDocumentRepository.deleteAll(toRemove);
entityLogEntryDocumentRepository.insert(toInsert);
entityLogDocumentRepository.save(newEntityLogDocument);
}
public void deleteEntityLog(String dossierId, String fileId) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogDocumentRepository.deleteById(entityLogId);
entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId);
}
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().addAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.insert(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList());
}
public void deleteEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().removeAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
private EntityLogDocument getEntityLogDocument(String entityLogId) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);
if (optionalEntityLogDocument.isEmpty()) {
throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId));
}
return optionalEntityLogDocument.get();
}
public Optional<EntityLog> findEntityLogByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId))
.map(EntityLogMongoService::fromDocument);
}
public boolean entityLogDocumentExists(String dossierId, String fileId) {
return entityLogDocumentRepository.existsById(EntityLogDocument.getDocumentId(dossierId, fileId));
}
public Optional<Integer> findLatestAnalysisNumber(String dossierId, String fileId) {
return entityLogDocumentRepository.findAnalysisNumberById(EntityLogDocument.getDocumentId(dossierId, fileId))
.map(EntityLogDocument::getAnalysisNumber);
}
public List<EntityLogEntry> findAllEntityLogEntriesWithManualChanges(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndManualChangesNotEmpty(EntityLogDocument.getDocumentId(dossierId, fileId))
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
public List<EntityLogEntry> findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(EntityLogDocument.getDocumentId(dossierId, fileId), analysisNumber)
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
public List<EntityLogEntry> findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogId(EntityLogDocument.getDocumentId(dossierId, fileId))
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
private static EntityLog fromDocument(EntityLogDocument entityLogDocument) {
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisVersion(entityLogDocument.getAnalysisVersion());
entityLog.setAnalysisNumber(entityLogDocument.getAnalysisNumber());
entityLog.setEntityLogEntry(entityLogDocument.getEntityLogEntryDocument()
.stream()
.map(EntityLogMongoService::fromDocument)
.collect(Collectors.toList()));
entityLog.setLegalBasis(entityLogDocument.getLegalBasis());
entityLog.setDictionaryVersion(entityLogDocument.getDictionaryVersion());
entityLog.setDossierDictionaryVersion(entityLogDocument.getDossierDictionaryVersion());
entityLog.setRulesVersion(entityLogDocument.getRulesVersion());
entityLog.setLegalBasisVersion(entityLogDocument.getLegalBasisVersion());
return entityLog;
}
private static EntityLogEntry fromDocument(EntityLogEntryDocument entityLogEntryDocument) {
EntityLogEntry entityLogEntry = new EntityLogEntry();
entityLogEntry.setId(entityLogEntryDocument.getEntryId());
entityLogEntry.setState(entityLogEntryDocument.getState());
entityLogEntry.setValue(entityLogEntryDocument.getValue());
entityLogEntry.setReason(entityLogEntryDocument.getReason());
entityLogEntry.setMatchedRule(entityLogEntryDocument.getMatchedRule());
entityLogEntry.setLegalBasis(entityLogEntryDocument.getLegalBasis());
entityLogEntry.setContainingNodeId(new ArrayList<>(entityLogEntryDocument.getContainingNodeId()));
entityLogEntry.setClosestHeadline(entityLogEntryDocument.getClosestHeadline());
entityLogEntry.setSection(entityLogEntryDocument.getSection());
entityLogEntry.setPositions(new ArrayList<>(entityLogEntryDocument.getPositions()));
entityLogEntry.setTextBefore(entityLogEntryDocument.getTextBefore());
entityLogEntry.setTextAfter(entityLogEntryDocument.getTextAfter());
entityLogEntry.setStartOffset(entityLogEntryDocument.getStartOffset());
entityLogEntry.setEndOffset(entityLogEntryDocument.getEndOffset());
entityLogEntry.setImageHasTransparency(entityLogEntryDocument.isImageHasTransparency());
entityLogEntry.setDictionaryEntry(entityLogEntryDocument.isDictionaryEntry());
entityLogEntry.setDossierDictionaryEntry(entityLogEntryDocument.isDossierDictionaryEntry());
entityLogEntry.setExcluded(entityLogEntryDocument.isExcluded());
entityLogEntry.setChanges(new ArrayList<>(entityLogEntryDocument.getChanges()));
entityLogEntry.setManualChanges(new ArrayList<>(entityLogEntryDocument.getManualChanges()));
entityLogEntry.setEngines(new HashSet<>(entityLogEntryDocument.getEngines()));
entityLogEntry.setReference(new HashSet<>(entityLogEntryDocument.getReference()));
entityLogEntry.setImportedRedactionIntersections(new HashSet<>(entityLogEntryDocument.getImportedRedactionIntersections()));
entityLogEntry.setNumberOfComments(entityLogEntryDocument.getNumberOfComments());
return entityLogEntry;
}
}

17
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogDocumentRepository.java Normal file
View File

@ -0,0 +1,17 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository;
import java.util.Optional;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
@Repository
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
@Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }")
Optional<EntityLogDocument> findAnalysisNumberById(@Param("_id") String id);
}

25
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/mongo/repository/EntityLogEntryDocumentRepository.java Normal file
View File

@ -0,0 +1,25 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository;
import java.util.List;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
@Repository
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {
@Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }")
List<EntityLogEntryDocument> findByEntityLogIdAndManualChangesNotEmpty(String entityLogId);
@Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }")
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber);
@Query("{ 'entityLogId' : ?0}")
List<EntityLogEntryDocument> findByEntityLogId(String entityLogId);
@Query(value = "{ 'entityLogId' : ?0}", delete = true)
void deleteByEntityLogId(String entityLogId);
}

1
persistence-service-v1/persistence-service-server-v1/build.gradle.kts
View File

@ -23,6 +23,7 @@ dependencies {
api(project(":persistence-service-external-api-impl-v2"))
api(project(":persistence-service-internal-api-impl-v1"))
api("com.iqser.red.commons:storage-commons:2.45.0")
api("com.knecon.fforesight:mongo-database-commons:maverick-mongo")
api("junit:junit:4.13.2")
api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
api("net.logstash.logback:logstash-logback-encoder:7.4")

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -21,6 +21,7 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import org.springframework.http.server.observation.DefaultServerRequestObservationConvention;
import org.springframework.http.server.observation.ServerRequestObservationContext;
import org.springframework.http.server.observation.ServerRequestObservationConvention;
@ -43,6 +44,7 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.databasetenantcommons.DatabaseTenantCommonsAutoConfiguration;
import com.knecon.fforesight.jobscommons.JobsAutoConfiguration;
import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
import com.knecon.fforesight.swaggercommons.SpringDocAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyMessagingConfiguration;
@ -62,7 +64,8 @@ import lombok.extern.slf4j.Slf4j;
@EnableScheduling
@EnableCaching
@EnableConfigurationProperties({FileManagementServiceSettings.class})
@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class})
@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, CassandraAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class})
@Import({PersistenceServiceExternalApiConfigurationV2.class, PersistenceServiceExternalApiConfiguration.class, PersistenceServiceInternalApiConfiguration.class, PersistenceServiceExternalApiCacheConfiguration.class, MultiTenancyWebConfiguration.class, PersistenceServiceProcessorConfiguration.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class})
public class Application implements ApplicationContextAware {

9
persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yaml
View File

@ -37,6 +37,15 @@ spring:
batch_size: 1000
order_inserts: true
order_updates: true
data:
mongodb:
auto-index-creation: true
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}
port: 27017
username: ${MONGODB_USER}
password: ${MONGODB_PASSWORD}
cache:
type: redis
mvc:

3
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java
View File

@ -64,9 +64,8 @@ public class FileTesterAndProvider {
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isGreaterThanOrEqualTo(1);
fileManagementStorageService.storeJSONObject(dossier.getId(),
fileManagementStorageService.saveEntityLog(dossier.getId(),
file.getFileId(),
FileType.ENTITY_LOG,
new EntityLog(1,
1,
List.of(EntityLogEntry.builder()

40
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ManualRedactionTest.java
View File

@ -139,7 +139,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyBoolean())).thenReturn(entityLog);
@ -333,7 +333,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -393,7 +393,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -448,7 +448,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -543,7 +543,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file1.getDossierId()), eq(file1.getFileId()), any(), anyBoolean())).thenReturn(entityLog1);
@ -562,7 +562,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -708,7 +708,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file1.getDossierId()), eq(file1.getFileId()), any(), anyBoolean())).thenReturn(entityLog1);
@ -727,7 +727,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -877,7 +877,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(file1.getDossierId(), file1.getFileId())).thenReturn(entityLog1);
@ -896,7 +896,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -1043,7 +1043,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier1.getId(), file1.getId(), FileType.ENTITY_LOG, entityLog1);
fileManagementStorageService.saveEntityLog(dossier1.getId(), file1.getId(), entityLog1);
var redactionRequest1 = RedactionRequest.builder().dossierId(file1.getDossierId()).fileId(file1.getFileId()).dossierTemplateId(file1.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(file1.getDossierId(), file1.getFileId())).thenReturn(entityLog1);
@ -1062,7 +1062,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier2.getId(), file2.getId(), FileType.ENTITY_LOG, entityLog2);
fileManagementStorageService.saveEntityLog(dossier2.getId(), file2.getId(), entityLog2);
var redactionRequest2 = RedactionRequest.builder().dossierId(file2.getDossierId()).fileId(file2.getFileId()).dossierTemplateId(file2.getDossierTemplateId()).build();
when(entityLogService.getEntityLog(eq(file2.getDossierId()), eq(file2.getFileId()), any(), anyBoolean())).thenReturn(entityLog2);
@ -1186,7 +1186,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1270,7 +1270,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1449,7 +1449,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1648,7 +1648,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
@ -1754,7 +1754,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(entityLog);
@ -1864,7 +1864,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any())).thenReturn(entityLog);
@ -1948,8 +1948,8 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
0,
0,
0);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.storeJSONObject(dossier.getId(), file.getId(), FileType.ENTITY_LOG, entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
when(entityLogService.getEntityLog(Mockito.any(), Mockito.any(), any(), anyBoolean())).thenReturn(entityLog);
var recatModel = RecategorizationRequestModel.builder()

189
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -1,42 +1,17 @@
package com.iqser.red.service.peristence.v1.server.integration.utils;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.integration.client.ApplicationConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.RedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient;
import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.redaction.v1.model.DroolsValidation;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener;
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.model.*;
import static org.mockito.Mockito.when;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.assertj.core.util.Lists;
import org.bson.BsonArray;
import org.bson.BsonDocument;
import org.bson.BsonString;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
@ -53,7 +28,11 @@ import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.StatementCallback;
import org.springframework.jdbc.datasource.SingleConnectionDataSource;
@ -69,13 +48,78 @@ import org.springframework.security.web.SecurityFilterChain;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.integration.client.ApplicationConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient;
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.RedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient;
import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.IndexInformationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationPreferencesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RuleSetRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ForceRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.LegalBasisChangeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ManualRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RecategorizationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.redaction.v1.model.DroolsValidation;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.databasetenantcommons.providers.TenantCreatedListener;
import com.knecon.fforesight.databasetenantcommons.providers.events.TenantCreatedEvent;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.tenantcommons.EncryptionDecryptionService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.model.AuthDetails;
import com.knecon.fforesight.tenantcommons.model.DatabaseConnection;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import com.knecon.fforesight.tenantcommons.model.S3StorageConnection;
import com.knecon.fforesight.tenantcommons.model.SearchConnection;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import com.mongodb.MongoCommandException;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoDatabase;
import static org.mockito.Mockito.when;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ExtendWith(SpringExtension.class)
@ -100,6 +144,12 @@ public abstract class AbstractPersistenceServerServiceTest {
@Autowired
protected StorageService storageService;
@Autowired
protected EntityLogMongoService entityLogMongoService;
@Autowired
protected EntityLogDocumentRepository entityLogDocumentRepository;
@Autowired
protected EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
@Autowired
protected FileManagementStorageService fileManagementStorageService;
@Autowired
protected DossierTemplateRepository dossierTemplateRepository;
@ -268,6 +318,7 @@ public abstract class AbstractPersistenceServerServiceTest {
private void createDefaultTenant() {
var postgreSQLContainerMaster = SpringPostgreSQLTestContainer.getInstance().withDatabaseName("integration-tests-db-master").withUsername("sa").withPassword("sa");
var mongoDbContainer = MongoDBTestContainer.getInstance();
var port = postgreSQLContainerMaster.getJdbcUrl().substring(0, postgreSQLContainerMaster.getJdbcUrl().lastIndexOf('/')).split(":")[3];
@ -296,6 +347,7 @@ public abstract class AbstractPersistenceServerServiceTest {
.numberOfShards("1")
.numberOfReplicas("5")
.build());
redactionTenant.setS3StorageConnection(S3StorageConnection.builder()
.key("key")
.secret("secret")
@ -304,6 +356,15 @@ public abstract class AbstractPersistenceServerServiceTest {
.region("eu")
.endpoint("endpoint")
.build());
redactionTenant.setMongoDBConnection(MongoDBConnection.builder()
.host(mongoDbContainer.getHost())
.port(String.valueOf(mongoDbContainer.getFirstMappedPort()))
.username(MongoDBTestContainer.MONGO_USERNAME)
.password(encryptionDecryptionService.encrypt(MongoDBTestContainer.MONGO_PASSWORD))
.database(MongoDBTestContainer.MONGO_DATABASE)
.build());
when(tenantsClient.getTenant("redaction")).thenReturn(redactionTenant);
when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant));
@ -332,6 +393,8 @@ public abstract class AbstractPersistenceServerServiceTest {
}
@AfterEach
public void cleanupStorage() {
@ -385,6 +448,9 @@ public abstract class AbstractPersistenceServerServiceTest {
indexInformationRepository.deleteAll();
applicationConfigRepository.deleteAll();
entityLogEntryDocumentRepository.deleteAll();
entityLogDocumentRepository.deleteAll();
});
}
@ -401,10 +467,18 @@ public abstract class AbstractPersistenceServerServiceTest {
var redisContainer = RedisTestContainer.getInstance();
redisContainer.start();
log.info("Hosts are - Redis: {}, Postgres: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost());
var mongoInstance = MongoDBTestContainer.getInstance();
mongoInstance.start();
createMongoDBDatabase(mongoInstance);
log.info("Hosts are - Redis: {}, Postgres: {}, MongoDB: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost(), mongoInstance.getHost());
TestPropertyValues.of("REDIS_PORT=" + redisContainer.getFirstMappedPort(),
"REDIS_HOST=" + redisContainer.getHost(),
"MONGODB_HOST=" + mongoInstance.getHost(),
"MONGODB_PORT=" + mongoInstance.getFirstMappedPort(),
"MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME,
"MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD,
"fforesight.jobs.enabled=false",
"fforesight.keycloak.enabled=false").applyTo(configurableApplicationContext.getEnvironment());
@ -412,11 +486,40 @@ public abstract class AbstractPersistenceServerServiceTest {
}
private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) {
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
MongoDBTestContainer.MONGO_USERNAME,
MongoDBTestContainer.MONGO_PASSWORD,
mongoDBTestContainer.getHost(),
mongoDBTestContainer.getFirstMappedPort()))) {
MongoDatabase database = mongoClient.getDatabase(MongoDBTestContainer.MONGO_DATABASE);
BsonDocument createUserCommand = new BsonDocument();
createUserCommand.append("createUser", new BsonString(MongoDBTestContainer.MONGO_USERNAME));
createUserCommand.append("pwd", new BsonString(MongoDBTestContainer.MONGO_PASSWORD));
BsonArray roles = new BsonArray();
roles.add(new BsonString("readWrite"));
createUserCommand.append("roles", roles);
try {
database.runCommand(createUserCommand);
} catch (MongoCommandException mongoCommandException) {
// ignore user already exists
if(mongoCommandException.getErrorCode() != 51003) {
throw mongoCommandException;
}
}
}
}
@Configuration
@EnableWebSecurity
@EnableMethodSecurity
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@ComponentScan("com.iqser.red.service.persistence")
@ComponentScan(basePackages = {"com.iqser.red.service.persistence" })
public static class TestConfiguration {
@Bean

35
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/MongoDBTestContainer.java Normal file
View File

@ -0,0 +1,35 @@
package com.iqser.red.service.peristence.v1.server.integration.utils;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.DockerImageName;
public final class MongoDBTestContainer extends GenericContainer<MongoDBTestContainer> {
private static final String IMAGE_VERSION = "mongo:7.0.2";
public static final Integer MONGO_PORT = 27017;
public static final String MONGO_DATABASE = "redaction";
public static final String MONGO_PASSWORD = "mongopassword";
public static final String MONGO_USERNAME = "mongousername";
private static MongoDBTestContainer mongoDB;
private MongoDBTestContainer() {
super(DockerImageName.parse(IMAGE_VERSION));
}
public static MongoDBTestContainer getInstance() {
if (mongoDB == null) {
mongoDB = new MongoDBTestContainer().withEnv("MONGO_INITDB_ROOT_USERNAME", MONGO_USERNAME)
.withEnv("MONGO_INITDB_ROOT_PASSWORD", MONGO_PASSWORD)
.withEnv("MONGO_INITDB_DATABASE", MONGO_DATABASE)
.withExposedPorts(MONGO_PORT);
}
return mongoDB;
}
}

10
persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml
View File

@ -19,7 +19,15 @@ spring:
order_inserts: true
order_updates: true
open-in-view: true
data:
mongodb:
auto-index-creation: true
# todo: multi-tenancy
database: redaction
host: ${MONGODB_HOST:localhost}
port: 27017
username: ${MONGODB_USER}
password: ${MONGODB_PASSWORD}
rabbitmq:
host: ${RABBITMQ_HOST:localhost}
port: ${RABBITMQ_PORT:5672}

2
persistence-service-v1/persistence-service-shared-api-v1/build.gradle.kts
View File

@ -3,6 +3,8 @@ plugins {
id("io.freefair.lombok") version "8.4"
}
val springBootStarterVersion = "3.1.5"
dependencies {
api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0")
api("com.google.guava:guava:31.1-jre")