RED-8702: Explore document databases to store entityLog

* new package shared-mongo for mongo related classes
* refactoring to include mapstruct for entitylog to entitylogdocument mapping
This commit is contained in:
maverickstuder 2024-03-26 11:56:40 +01:00
parent 4eb321c1d1
commit d79ca13091
22 changed files with 432 additions and 366 deletions

View File

@ -8,6 +8,7 @@ val springCloudVersion = "4.0.4"
dependencies {
api(project(":persistence-service-shared-api-v1"))
api(project(":persistence-service-shared-mongo-v1"))
api(project(":persistence-service-external-api-v1"))
api(project(":persistence-service-internal-api-v1"))
api("com.iqser.red.service:pdftron-redaction-service-api-v1:${rootProject.extra.get("pdftronRedactionServiceVersion")}") {
@ -35,7 +36,6 @@ dependencies {
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.knecon.fforesight:mongo-database-commons:maverick-mongo6")
api("com.knecon.fforesight:database-tenant-commons:maverick-mongo")
api("com.knecon.fforesight:keycloak-commons:maverick-mongo")
api("com.knecon.fforesight:tracing-commons:0.5.0")
@ -45,7 +45,6 @@ dependencies {
api("org.springframework.boot:spring-boot-starter-mail:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-jpa:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-web:${springBootStarterVersion}")
api("com.iqser.red.commons:spring-commons:2.1.0")

View File

@ -5,6 +5,8 @@ import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.cloud.openfeign.support.PageJacksonModule;
import org.springframework.cloud.openfeign.support.SortJacksonModule;
@ -12,6 +14,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.env.Environment;
import org.springframework.jdbc.core.JdbcTemplate;
@ -25,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.client.redactio
import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient;
import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
@ -33,6 +37,7 @@ import lombok.extern.slf4j.Slf4j;
@Configuration
@ComponentScan
@EnableFeignClients(basePackageClasses = {PDFTronClient.class, StatusReportClient.class, SearchClient.class, RedactionClient.class, UsersClient.class})
@ImportAutoConfiguration(SharedMongoAutoConfiguration.class)
public class PersistenceServiceProcessorConfiguration {
public static final String TENANT_DATA_SOURCE_QUALIFIER = "multiTenantDataSource";

View File

@ -1,76 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.document;
import java.util.ArrayList;
import java.util.List;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@Document(collection = "entity-logs")
public class EntityLogDocument {
@Id
@EqualsAndHashCode.Include
private String id;
private String dossierId;
private String fileId;
private long analysisVersion;
private int analysisNumber;
@DBRef
private List<EntityLogEntryDocument> entityLogEntryDocument = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
public EntityLogDocument(String dossierId, String fileId, EntityLog entityLog) {
this.id = getDocumentId(dossierId, fileId);
this.dossierId = dossierId;
this.fileId = fileId;
this.entityLogEntryDocument = new ArrayList<>(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(this.id, entityLogEntry))
.toList());
this.setAnalysisVersion(entityLog.getAnalysisVersion());
this.setAnalysisNumber(entityLog.getAnalysisNumber());
this.setLegalBasis(entityLog.getLegalBasis());
this.setDictionaryVersion(entityLog.getDictionaryVersion());
this.setDossierDictionaryVersion(entityLog.getDossierDictionaryVersion());
this.setRulesVersion(entityLog.getRulesVersion());
this.setLegalBasisVersion(entityLog.getLegalBasisVersion());
}
public static String getDocumentId(String dossierId, String fileId) {
return dossierId + "/" + fileId;
}
}

View File

@ -12,7 +12,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -20,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.storage.commons.exception.StorageException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;

View File

@ -1,240 +0,0 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.management.v1.processor.exception.EntityLogDocumentNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@Service
public class EntityLogMongoService {
private final EntityLogDocumentRepository entityLogDocumentRepository;
private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository, EntityLogEntryDocumentRepository entityLogEntryDocumentRepository) {
this.entityLogDocumentRepository = entityLogDocumentRepository;
this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
}
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog));
entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
.toList());
}
// this does everything : insert when not found and update if found
// todo: remove and replace when services use insert,update,delete correctly
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId));
if (optionalEntityLogDocument.isEmpty()) {
// throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId));
insertEntityLog(dossierId, fileId, entityLog);
return;
}
EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get();
List<EntityLogEntryDocument> oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument();
EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog);
List<EntityLogEntryDocument> newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument();
List<EntityLogEntryDocument> toUpdate = new ArrayList<>(newEntityLogEntryDocuments);
toUpdate.retainAll(oldEntityLogEntryDocuments);
List<EntityLogEntryDocument> toRemove = new ArrayList<>(oldEntityLogEntryDocuments);
toRemove.removeAll(toUpdate);
List<EntityLogEntryDocument> toInsert = new ArrayList<>(newEntityLogEntryDocuments);
toInsert.removeAll(toUpdate);
entityLogEntryDocumentRepository.saveAll(toUpdate);
entityLogEntryDocumentRepository.deleteAll(toRemove);
entityLogEntryDocumentRepository.insert(toInsert);
entityLogDocumentRepository.save(newEntityLogDocument);
}
public void deleteEntityLog(String dossierId, String fileId) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogDocumentRepository.deleteById(entityLogId);
entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId);
}
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().addAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.insert(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList());
}
public void deleteEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> new EntityLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocument().removeAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
private EntityLogDocument getEntityLogDocument(String entityLogId) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);
if (optionalEntityLogDocument.isEmpty()) {
throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId));
}
return optionalEntityLogDocument.get();
}
public Optional<EntityLog> findEntityLogByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId))
.map(EntityLogMongoService::fromDocument);
}
public boolean entityLogDocumentExists(String dossierId, String fileId) {
return entityLogDocumentRepository.existsById(EntityLogDocument.getDocumentId(dossierId, fileId));
}
public Optional<Integer> findLatestAnalysisNumber(String dossierId, String fileId) {
return entityLogDocumentRepository.findAnalysisNumberById(EntityLogDocument.getDocumentId(dossierId, fileId))
.map(EntityLogDocument::getAnalysisNumber);
}
public List<EntityLogEntry> findAllEntityLogEntriesWithManualChanges(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndManualChangesNotEmpty(EntityLogDocument.getDocumentId(dossierId, fileId))
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
public List<EntityLogEntry> findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(EntityLogDocument.getDocumentId(dossierId, fileId), analysisNumber)
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
public List<EntityLogEntry> findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogId(EntityLogDocument.getDocumentId(dossierId, fileId))
.stream()
.map(EntityLogMongoService::fromDocument)
.toList();
}
private static EntityLog fromDocument(EntityLogDocument entityLogDocument) {
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisVersion(entityLogDocument.getAnalysisVersion());
entityLog.setAnalysisNumber(entityLogDocument.getAnalysisNumber());
entityLog.setEntityLogEntry(entityLogDocument.getEntityLogEntryDocument()
.stream()
.map(EntityLogMongoService::fromDocument)
.collect(Collectors.toList()));
entityLog.setLegalBasis(entityLogDocument.getLegalBasis());
entityLog.setDictionaryVersion(entityLogDocument.getDictionaryVersion());
entityLog.setDossierDictionaryVersion(entityLogDocument.getDossierDictionaryVersion());
entityLog.setRulesVersion(entityLogDocument.getRulesVersion());
entityLog.setLegalBasisVersion(entityLogDocument.getLegalBasisVersion());
return entityLog;
}
private static EntityLogEntry fromDocument(EntityLogEntryDocument entityLogEntryDocument) {
EntityLogEntry entityLogEntry = new EntityLogEntry();
entityLogEntry.setId(entityLogEntryDocument.getEntryId());
entityLogEntry.setType(entityLogEntryDocument.getType());
entityLogEntry.setEntryType(entityLogEntryDocument.getEntryType());
entityLogEntry.setState(entityLogEntryDocument.getState());
entityLogEntry.setValue(entityLogEntryDocument.getValue());
entityLogEntry.setReason(entityLogEntryDocument.getReason());
entityLogEntry.setMatchedRule(entityLogEntryDocument.getMatchedRule());
entityLogEntry.setLegalBasis(entityLogEntryDocument.getLegalBasis());
entityLogEntry.setContainingNodeId(new ArrayList<>(entityLogEntryDocument.getContainingNodeId()));
entityLogEntry.setClosestHeadline(entityLogEntryDocument.getClosestHeadline());
entityLogEntry.setSection(entityLogEntryDocument.getSection());
entityLogEntry.setPositions(new ArrayList<>(entityLogEntryDocument.getPositions()));
entityLogEntry.setTextBefore(entityLogEntryDocument.getTextBefore());
entityLogEntry.setTextAfter(entityLogEntryDocument.getTextAfter());
entityLogEntry.setStartOffset(entityLogEntryDocument.getStartOffset());
entityLogEntry.setEndOffset(entityLogEntryDocument.getEndOffset());
entityLogEntry.setImageHasTransparency(entityLogEntryDocument.isImageHasTransparency());
entityLogEntry.setDictionaryEntry(entityLogEntryDocument.isDictionaryEntry());
entityLogEntry.setDossierDictionaryEntry(entityLogEntryDocument.isDossierDictionaryEntry());
entityLogEntry.setExcluded(entityLogEntryDocument.isExcluded());
entityLogEntry.setChanges(new ArrayList<>(entityLogEntryDocument.getChanges()));
entityLogEntry.setManualChanges(new ArrayList<>(entityLogEntryDocument.getManualChanges()));
entityLogEntry.setEngines(new HashSet<>(entityLogEntryDocument.getEngines()));
entityLogEntry.setReference(new HashSet<>(entityLogEntryDocument.getReference()));
entityLogEntry.setImportedRedactionIntersections(new HashSet<>(entityLogEntryDocument.getImportedRedactionIntersections()));
entityLogEntry.setNumberOfComments(entityLogEntryDocument.getNumberOfComments());
return entityLogEntry;
}
}

View File

@ -22,6 +22,7 @@ dependencies {
api(project(":persistence-service-external-api-impl-v1"))
api(project(":persistence-service-external-api-impl-v2"))
api(project(":persistence-service-internal-api-impl-v1"))
api(project(":persistence-service-shared-mongo-v1"))
api("com.iqser.red.commons:storage-commons:2.45.0")
api("junit:junit:4.13.2")
api("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")

View File

@ -48,6 +48,7 @@ import com.knecon.fforesight.databasetenantcommons.DatabaseTenantCommonsAutoConf
import com.knecon.fforesight.jobscommons.JobsAutoConfiguration;
import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
import com.knecon.fforesight.swaggercommons.SpringDocAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyMessagingConfiguration;
@ -68,6 +69,7 @@ import lombok.extern.slf4j.Slf4j;
@EnableCaching
@EnableConfigurationProperties({FileManagementServiceSettings.class})
@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
@EnableMongoLiquibase
@ImportAutoConfiguration({StorageAutoConfiguration.class, JobsAutoConfiguration.class, DatabaseTenantCommonsAutoConfiguration.class, MultiTenancyAutoConfiguration.class, SpringDocAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, CassandraAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
@Import({PersistenceServiceExternalApiConfigurationV2.class, PersistenceServiceExternalApiConfiguration.class, PersistenceServiceInternalApiConfiguration.class, PersistenceServiceExternalApiCacheConfiguration.class, MultiTenancyWebConfiguration.class, PersistenceServiceProcessorConfiguration.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class})

View File

@ -0,0 +1,63 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Objects;
import java.util.Optional;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper.EntityLogMapper;
import lombok.SneakyThrows;
public class EntityLogMapperTest {
private final EntityLogMapper mapper = EntityLogMapper.INSTANCE;
private final String ENTITY_LOG = "files/entity-log/b2cbdd4dca0aa1aa0ebbfc5cc1462df0.ENTITY_LOG.json";
private static final String TEST_DOSSIER_ID = "91ce8e90-9aec-473c-b8c3-cbe16443ad34";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
@Test
@SneakyThrows
public void testEntityLogMapper() {
var file = new ClassPathResource(String.format(ENTITY_LOG));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLogBefore = objectMapper.readValue(file.getInputStream(), EntityLog.class);
EntityLogDocument entityLogDocument = mapper.toLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLogBefore);
assertEquals(entityLogDocument.getDossierId(), TEST_DOSSIER_ID);
assertEquals(entityLogDocument.getFileId(), TEST_FILE_ID);
assertEquals(entityLogDocument.getId(), mapper.getLogId(TEST_DOSSIER_ID, TEST_FILE_ID));
Optional<EntityLogEntryDocument> optionalEntityLogEntryDocument = entityLogDocument.getEntityLogEntryDocuments()
.stream()
.findFirst();
assertTrue(optionalEntityLogEntryDocument.isPresent());
assertNotNull(optionalEntityLogEntryDocument.get().getEntityLogId());
assertNotNull(optionalEntityLogEntryDocument.get().getEntryId());
assertEquals(optionalEntityLogEntryDocument.get().getId(),
mapper.getLogEntryId(mapper.getLogId(TEST_DOSSIER_ID, TEST_FILE_ID), optionalEntityLogEntryDocument.get().getEntryId()));
EntityLog entityLogAfter = mapper.fromLogDocument(entityLogDocument);
assertEquals(entityLogBefore, entityLogAfter);
}
}

View File

@ -13,9 +13,9 @@ import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;

View File

@ -65,9 +65,6 @@ import com.iqser.red.service.persistence.management.v1.processor.roles.Applicati
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
@ -101,6 +98,9 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.users.UserService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.model.DroolsValidation;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;

View File

@ -0,0 +1,23 @@
plugins {
id("com.iqser.red.service.java-conventions")
id("io.freefair.lombok") version "8.4"
}
val springBootStarterVersion = "3.1.5"
dependencies {
api(project(":persistence-service-shared-api-v1"))
api("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0")
api("com.google.guava:guava:31.1-jre")
api("com.knecon.fforesight:mongo-database-commons:1.0-SNAPSHOT")
api("org.springframework.boot:spring-boot-starter-data-mongodb:${springBootStarterVersion}")
api("org.springframework.boot:spring-boot-starter-validation:3.1.3")
testImplementation("com.iqser.red.commons:test-commons:2.1.0")
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
compileOnly("org.springdoc:springdoc-openapi-ui:1.7.0")
implementation("org.mapstruct:mapstruct:1.5.5.Final")
annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final")
}
description = "persistence-service-shared-mongo-v1"

View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.service.v1.api.shared.mongo;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
@ComponentScan
@AutoConfiguration
public class SharedMongoAutoConfiguration {
}

View File

@ -0,0 +1,48 @@
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.document;
import java.util.ArrayList;
import java.util.List;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@Document(collection = "entity-logs")
public class EntityLogDocument {
@Id
@EqualsAndHashCode.Include
private String id;
private String dossierId;
private String fileId;
private long analysisVersion;
private int analysisNumber;
@DBRef
private List<EntityLogEntryDocument> entityLogEntryDocuments = new ArrayList<>();
private List<EntityLogLegalBasis> legalBasis = new ArrayList<>();
private long dictionaryVersion = -1;
private long dossierDictionaryVersion = -1;
private long rulesVersion = -1;
private long legalBasisVersion = -1;
}

View File

@ -1,4 +1,4 @@
package com.iqser.red.service.persistence.management.v1.processor.document;
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.document;
import java.util.ArrayList;
import java.util.HashSet;
@ -73,41 +73,4 @@ public class EntityLogEntryDocument {
Set<String> importedRedactionIntersections = new HashSet<>();
int numberOfComments;
public EntityLogEntryDocument(String entityLogId, EntityLogEntry entityLogEntry) {
this.id = entityLogId + "/" + entityLogEntry.getId();
this.entryId = entityLogEntry.getId();
this.entityLogId = entityLogId;
this.setType(entityLogEntry.getType());
this.setEntryType(entityLogEntry.getEntryType());
this.setState(entityLogEntry.getState());
this.setValue(entityLogEntry.getValue());
this.setReason(entityLogEntry.getReason());
this.setMatchedRule(entityLogEntry.getMatchedRule());
this.setLegalBasis(entityLogEntry.getLegalBasis());
this.setContainingNodeId(entityLogEntry.getContainingNodeId());
this.setClosestHeadline(entityLogEntry.getClosestHeadline());
this.setSection(entityLogEntry.getSection());
this.setPositions(entityLogEntry.getPositions());
this.setTextBefore(entityLogEntry.getTextBefore());
this.setTextAfter(entityLogEntry.getTextAfter());
this.setStartOffset(entityLogEntry.getStartOffset());
this.setEndOffset(entityLogEntry.getEndOffset());
this.setImageHasTransparency(entityLogEntry.isImageHasTransparency());
this.setDictionaryEntry(entityLogEntry.isDictionaryEntry());
this.setDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry());
this.setExcluded(entityLogEntry.isExcluded());
this.setChanges(entityLogEntry.getChanges());
this.setManualChanges(entityLogEntry.getManualChanges());
this.setEngines(entityLogEntry.getEngines());
this.setReference(entityLogEntry.getReference());
this.setImportedRedactionIntersections(entityLogEntry.getImportedRedactionIntersections());
this.setNumberOfComments(entityLogEntry.getNumberOfComments());
}
}

View File

@ -1,4 +1,4 @@
package com.iqser.red.service.persistence.management.v1.processor.exception;
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.exception;
public class EntityLogDocumentNotFoundException extends RuntimeException {

View File

@ -0,0 +1,62 @@
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.mapstruct.factory.Mappers;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
@Mapper
public interface EntityLogMapper {
EntityLogMapper INSTANCE = Mappers.getMapper(EntityLogMapper.class);
@Mapping(target = "entityLogEntry", source = "entityLogEntryDocuments")
EntityLog fromLogDocument(EntityLogDocument entityLogDocument);
@Mapping(target = "id", source = "entityLogEntryDocument.entryId")
EntityLogEntry fromLogEntryDocument(EntityLogEntryDocument entityLogEntryDocument);
List<EntityLogEntryDocument> toLogEntryDocuments(List<EntityLogEntry> entityLogEntries);
@Mapping(target = "id", expression = "java(getLogId(dossierId, fileId))")
@Mapping(target = "entityLogEntryDocuments", expression ="java(toEntryDocumentList(getLogId(dossierId, fileId), entityLog.getEntityLogEntry()))")
EntityLogDocument toLogDocument(String dossierId, String fileId, EntityLog entityLog);
@Mapping(target = "id", expression = "java(getLogEntryId(entityLogId, entityLogEntry.getId()))")
@Mapping(target = "entryId", source = "entityLogEntry.id")
EntityLogEntryDocument toLogEntryDocument(String entityLogId, EntityLogEntry entityLogEntry);
default List<EntityLogEntryDocument> toEntryDocumentList(String entityLogId, List<EntityLogEntry> entityLogEntries) {
return entityLogEntries.stream()
.map(entityLogEntry -> toLogEntryDocument(entityLogId, entityLogEntry))
.collect(Collectors.toList());
}
default String getLogId(String dossierId, String fileId) {
return dossierId + "/" + fileId;
}
default String getLogEntryId(String entityLogId, String entryId) {
return entityLogId + "/" + entryId;
}
}

View File

@ -1,4 +1,4 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository;
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository;
import java.util.Optional;
@ -7,7 +7,7 @@ import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument;
@Repository
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {

View File

@ -1,4 +1,4 @@
package com.iqser.red.service.persistence.management.v1.processor.service.mongo.repository;
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository;
import java.util.List;
@ -6,7 +6,7 @@ import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
@Repository
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {

View File

@ -0,0 +1,188 @@
package com.iqser.red.service.persistence.service.v1.api.shared.mongo.service;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.document.EntityLogEntryDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.exception.EntityLogDocumentNotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.mapper.EntityLogMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
@Service
public class EntityLogMongoService {
private final EntityLogDocumentRepository entityLogDocumentRepository;
private final EntityLogEntryDocumentRepository entityLogEntryDocumentRepository;
private final EntityLogMapper mapper = EntityLogMapper.INSTANCE;
public EntityLogMongoService(EntityLogDocumentRepository entityLogDocumentRepository, EntityLogEntryDocumentRepository entityLogEntryDocumentRepository) {
this.entityLogDocumentRepository = entityLogDocumentRepository;
this.entityLogEntryDocumentRepository = entityLogEntryDocumentRepository;
}
public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(mapper.toLogDocument(dossierId, fileId, entityLog));
entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry()
.stream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
.toList());
}
// this does everything : insert when not found and update if found
// todo: remove and replace when services use insert,update,delete correctly
public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(mapper.getLogId(dossierId, fileId));
if (optionalEntityLogDocument.isEmpty()) {
// throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId));
insertEntityLog(dossierId, fileId, entityLog);
return;
}
EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get();
List<EntityLogEntryDocument> oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocuments();
EntityLogDocument newEntityLogDocument = mapper.toLogDocument(dossierId, fileId, entityLog);
List<EntityLogEntryDocument> newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocuments();
List<EntityLogEntryDocument> toUpdate = new ArrayList<>(newEntityLogEntryDocuments);
toUpdate.retainAll(oldEntityLogEntryDocuments);
List<EntityLogEntryDocument> toRemove = new ArrayList<>(oldEntityLogEntryDocuments);
toRemove.removeAll(toUpdate);
List<EntityLogEntryDocument> toInsert = new ArrayList<>(newEntityLogEntryDocuments);
toInsert.removeAll(toUpdate);
entityLogEntryDocumentRepository.saveAll(toUpdate);
entityLogEntryDocumentRepository.deleteAll(toRemove);
entityLogEntryDocumentRepository.insert(toInsert);
entityLogDocumentRepository.save(newEntityLogDocument);
}
public void deleteEntityLog(String dossierId, String fileId) {
String entityLogId = mapper.getLogId(dossierId, fileId);
entityLogDocumentRepository.deleteById(entityLogId);
entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId);
}
public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = mapper.getLogId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocuments().addAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.insert(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = mapper.getLogId(dossierId, fileId);
entityLogEntryDocumentRepository.saveAll(entityLogEntries.stream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.toList());
}
public void deleteEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
String entityLogId = mapper.getLogId(dossierId, fileId);
EntityLogDocument entityLogDocument = getEntityLogDocument(entityLogId);
List<EntityLogEntryDocument> entityLogEntryDocuments = entityLogEntries.stream()
.map(entityLogEntry -> mapper.toLogEntryDocument(entityLogId, entityLogEntry))
.toList();
entityLogDocument.getEntityLogEntryDocuments().removeAll(entityLogEntryDocuments);
entityLogEntryDocumentRepository.deleteAll(entityLogEntryDocuments);
entityLogDocumentRepository.save(entityLogDocument);
}
private EntityLogDocument getEntityLogDocument(String entityLogId) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);
if (optionalEntityLogDocument.isEmpty()) {
throw new EntityLogDocumentNotFoundException(String.format("Entity log not found for %s", entityLogId));
}
return optionalEntityLogDocument.get();
}
public Optional<EntityLog> findEntityLogByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogDocumentRepository.findById(mapper.getLogId(dossierId, fileId))
.map(mapper::fromLogDocument);
}
public boolean entityLogDocumentExists(String dossierId, String fileId) {
return entityLogDocumentRepository.existsById(mapper.getLogId(dossierId, fileId));
}
public Optional<Integer> findLatestAnalysisNumber(String dossierId, String fileId) {
return entityLogDocumentRepository.findAnalysisNumberById(mapper.getLogId(dossierId, fileId))
.map(EntityLogDocument::getAnalysisNumber);
}
public List<EntityLogEntry> findAllEntityLogEntriesWithManualChanges(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndManualChangesNotEmpty(mapper.getLogId(dossierId, fileId))
.stream()
.map(mapper::fromLogEntryDocument)
.toList();
}
public List<EntityLogEntry> findAllEntityLogEntriesByAnalysisNumber(String dossierId, String fileId, int analysisNumber) {
return entityLogEntryDocumentRepository.findByEntityLogIdAndChangesAnalysisNumber(mapper.getLogId(dossierId, fileId), analysisNumber)
.stream()
.map(mapper::fromLogEntryDocument)
.toList();
}
public List<EntityLogEntry> findAllEntriesByDossierIdAndFileId(String dossierId, String fileId) {
return entityLogEntryDocumentRepository.findByEntityLogId(mapper.getLogId(dossierId, fileId))
.stream()
.map(mapper::fromLogEntryDocument)
.toList();
}
}

View File

@ -0,0 +1,16 @@
package com.iqser.red.service.persistence.service.v1.api.shared.mongo;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
public class IdentityTest {
@Test
public void mockTest() {
int i = 1;
assertThat(i).isEqualTo(1);
}
}

View File

@ -8,6 +8,7 @@ include(":persistence-service-internal-api-impl-v1")
include(":persistence-service-external-api-v2")
include(":persistence-service-external-api-impl-v2")
include(":persistence-service-internal-api-v1")
include(":persistence-service-shared-mongo-v1")
project(":persistence-service-processor-v1").projectDir = file("persistence-service-v1/persistence-service-processor-v1")
project(":persistence-service-shared-api-v1").projectDir = file("persistence-service-v1/persistence-service-shared-api-v1")
project(":persistence-service-external-api-impl-v1").projectDir = file("persistence-service-v1/persistence-service-external-api-impl-v1")
@ -17,3 +18,4 @@ project(":persistence-service-internal-api-impl-v1").projectDir = file("persiste
project(":persistence-service-external-api-v2").projectDir = file("persistence-service-v1/persistence-service-external-api-v2")
project(":persistence-service-external-api-impl-v2").projectDir = file("persistence-service-v1/persistence-service-external-api-impl-v2")
project(":persistence-service-internal-api-v1").projectDir = file("persistence-service-v1/persistence-service-internal-api-v1")
project(":persistence-service-shared-mongo-v1").projectDir = file("persistence-service-v1/persistence-service-shared-mongo-v1")