Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* now using shared mongo
This commit is contained in:
maverickstuder 2024-03-26 12:09:51 +01:00
parent 04418f57d3
commit e499caa6c6
19 changed files with 13 additions and 337840 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java
View File

@ -17,6 +17,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
@ -28,11 +29,11 @@ import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.aop.ObservedAspect;
@EnableCaching
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@EnableMongoRepositories
@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
public class Application {

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogDocumentNotFoundException.java
View File

@ -1,10 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
public class EntityLogDocumentNotFoundException extends RuntimeException {
public EntityLogDocumentNotFoundException(String errorMessage) {
super(errorMessage);
}
}

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogDocumentRepository.java
View File

@ -1,15 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.Optional;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.data.repository.query.Param;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
public interface EntityLogDocumentRepository extends MongoRepository<EntityLogDocument, String> {
@Query(value = "{ '_id' : ?0 }", fields = "{ 'analysisNumber' : 1 }")
Optional<EntityLogDocument> findAnalysisNumberById(@Param("_id") String id);
}

23
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogEntryDocumentRepository.java
View File

@ -1,23 +0,0 @@
package com.iqser.red.service.redaction.v1.server.mongo;
import java.util.List;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogEntryDocument;
public interface EntityLogEntryDocumentRepository extends MongoRepository<EntityLogEntryDocument, String> {
@Query("{ 'entityLogId' : ?0, 'manualChanges' : { $exists: true, $not: { $size: 0 } } }")
List<EntityLogEntryDocument> findByEntityLogIdAndManualChangesNotEmpty(String entityLogId);
@Query("{ 'entityLogId' : ?0, 'changes.analysisNumber' : ?1 }")
List<EntityLogEntryDocument> findByEntityLogIdAndChangesAnalysisNumber(String entityLogId, int analysisNumber);
@Query("{ 'entityLogId' : ?0}")
List<EntityLogEntryDocument> findByEntityLogId(String entityLogId);
@Query(value = "{ 'entityLogId' : ?0}", delete = true)
void deleteByEntityLogId(String entityLogId);
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -14,10 +14,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;

8
redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml
View File

@ -53,7 +53,13 @@ spring:
host: ${MONGODB_HOST:localhost}
port: 27017
username: ${MONGODB_USER}
password: ${MONGODB_PASSWORD}
password: ${MONGODB_PASSWORD}spring:
autoconfigure:
exclude: org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration,
org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration,
org.springframework.boot.autoconfigure.data.mongo.MongoReactiveDataAutoConfiguration,
org.springframework.boot.autoconfigure.data.mongo.MongoReactiveRepositoriesAutoConfiguration,
org.springframework.boot.autoconfigure.data.mongo.MongoRepositoriesAutoConfiguration
management:
endpoint:

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -39,13 +39,13 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogDocumentRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.repository.EntityLogEntryDocumentRepository;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogDocumentRepository;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogEntryDocumentRepository;
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;

52
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoLiquibaseTest.java
View File

@ -1,52 +0,0 @@
package com.iqser.red.service.redaction.v1.server;
import java.io.File;
import org.bson.Document;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.io.ResourceLoader;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import liquibase.Liquibase;
import liquibase.database.DatabaseFactory;
import liquibase.ext.mongodb.database.MongoLiquibaseDatabase;
import liquibase.resource.ClassLoaderResourceAccessor;
import lombok.SneakyThrows;
public class MyMongoLiquibaseTest {
private static final String HOSTNAME = "localhost";
private static final String PORT = "27017";
private static final String DATABASE = "redaction";
private static final String USERNAME = "root";
private static final String PASSWORD = "XaRIBExTr8JrCSfgaHek3dl7wJj7dKZm";
private static final String URL = String.format("mongodb://%s:%s/%s", HOSTNAME, PORT, DATABASE);
@Test
//@Disabled
@SneakyThrows
public void testMongoDB() {
try (MongoLiquibaseDatabase database = (MongoLiquibaseDatabase) DatabaseFactory.getInstance().openDatabase(URL, USERNAME, PASSWORD, null, null)) {
//MongoCollection<Document> logs = database.getMongoDatabase().getCollection("entity-logs");
//long count = logs.countDocuments();
String path = "/mongo/liquibase/changelog/mongo.changelog-tenant.xml";
try (Liquibase liquibase = new Liquibase(path, new ClassLoaderResourceAccessor(), database)) {
liquibase.update("");
}
}
}
}

101
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java
View File

@ -1,101 +0,0 @@
package com.iqser.red.service.redaction.v1.server;
import static com.mongodb.client.model.Filters.eq;
import org.bson.BsonArray;
import org.bson.BsonDocument;
import org.bson.BsonString;
import org.bson.Document;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.data.mongodb.core.MongoTemplate;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
@Disabled
public class MyMongoTest {
private static final String HOSTNAME = "localhost";
private static final String PORT = "27017";
private static final String DATABASE = "redaction";
private static final String USERNAME = "root";
private static final String PASSWORD = "AP8ckozpAl55JDAAFR4rxpbPj1WXdBTL";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
public static final String COLLECTION_ENTITY_LOG = "entity-logs";
private static final String CONNECTION_STRING = String.format("mongodb://%s:%s@%s:%s/", USERNAME, PASSWORD, HOSTNAME, PORT);
@Test
public void deleteMongoDBDatabase() {
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
USERNAME,
PASSWORD,
HOSTNAME,
PORT))) {
mongoClient.getDatabase("new_database").drop();
}
}
@Test
public void createDatabaseUser() {
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
USERNAME,
PASSWORD,
HOSTNAME,
PORT))) {
MongoDatabase database = mongoClient.getDatabase("last_database");
BsonDocument createUserCommand = new BsonDocument();
createUserCommand.append("createUser", new BsonString(USERNAME));
createUserCommand.append("pwd", new BsonString(PASSWORD));
BsonArray roles = new BsonArray();
roles.add(new BsonString("readWrite"));
createUserCommand.append("roles", roles);
database.runCommand(createUserCommand);
}
}
@Test
public void testMongoDB() {
try (MongoClient mongoClient = MongoClients.create(CONNECTION_STRING)) {
MongoDatabase database = mongoClient.getDatabase(DATABASE);
MongoCollection<Document> collection = database.getCollection(COLLECTION_ENTITY_LOG);
Document doc = collection.find(eq("_id", TEST_FILE_ID)).first();
if (doc != null) {
System.out.println("Found doc with _id: " + doc.get("_id"));
} else {
System.out.println("No matching documents found.");
}
}
}
@Test
public void testMongoDB_Template() {
try (MongoClient mongoClient = MongoClients.create(CONNECTION_STRING)) {
MongoTemplate mongoTemplate = new MongoTemplate(mongoClient, DATABASE);
MongoCollection<Document> collection = mongoTemplate.getCollection(COLLECTION_ENTITY_LOG);
Document doc = collection.find(eq("_id", TEST_FILE_ID)).first();
if (doc != null) {
System.out.println("Found doc with _id: " + doc.get("_id"));
} else {
System.out.println("No matching documents found.");
}
}
}
}

398
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java
View File

@ -1,398 +0,0 @@
package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.runtime.KieContainer;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.mongo.EntityLogMongoService;
import com.iqser.red.storage.commons.service.StorageServiceImpl;
import com.knecon.fforesight.mongo.database.commons.config.MongoDbConfiguration;
import com.knecon.fforesight.mongo.database.commons.config.MultiTenantMongoDBFactory;
import com.knecon.fforesight.mongo.database.commons.service.MongoClientCache;
import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProviderImpl;
import com.knecon.fforesight.mongo.database.commons.service.MongoDataSources;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(MyOtherMongoTest.MongoTestConfiguration.class)
@ContextConfiguration(initializers = {MyOtherMongoTest.Initializer.class})
@Disabled
public class MyOtherMongoTest {
@MockBean
private RabbitTemplate rabbitTemplate;
@MockBean
private TenantsClient tenantsClient;
@MockBean
protected KieContainer kieContainer;
@MockBean
protected DictionaryClient dictionaryClient;
private static final String HOSTNAME = "localhost";
private static final String PORT = "27017";
private static final String DATABASE = "redaction";
private static final String USERNAME = "root";
private static final String PASSWORD = "AP8ckozpAl55JDAAFR4rxpbPj1WXdBTL";
private static final String TEST_DOSSIER_2_ID = "bigentestdossierid";
private static final String TEST_FILE_2_ID = "bigentityfileid";
private static final String TEST_FILE_3_ID = "upd_bigentityfileid";
private static final String TEST_DOSSIER_ID = "testdossierid";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
public static final String COLLECTION_ENTITY_LOG = "entity-logs";
private static final String CONNECTION_STRING = String.format("mongodb://%s:%s@%s:%s/", USERNAME, PASSWORD, HOSTNAME, PORT);
@Autowired
private StorageServiceImpl storageService;
@Autowired
private EntityLogMongoService entityLogMongoService;
@MockBean
private MongoConnectionProvider mongoConnectionProvider;
@BeforeEach
public void setupTenantAndMockDBConnection() {
TenantContext.setTenantId("redaction");
when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder()
.host("localhost")
.port("27017")
.database("redaction")
.username(USERNAME)
.password(PASSWORD)
.build());
}
@Test
@SneakyThrows
public void testUpdateBigEntityLogDocumentByAddingOne() {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLogEntry entityLogEntry = objectMapper.readValue(NEW_ENTITY_LOG_ENTRY, EntityLogEntry.class);
entityLogMongoService.insertEntityLogEntries(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, List.of(entityLogEntry));
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getEntityLogEntry().size(), 1707);
}
@Test
@SneakyThrows
public void testUpdateBigEntityLogDocumentByNewLog() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_3_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLog.setAnalysisNumber(entityLog.getAnalysisNumber() + 1);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getAnalysisNumber(), entityLog.getAnalysisNumber());
assertEquals(found.get().getEntityLogEntry().size(), 1707);
}
@Test
@SneakyThrows
public void testUpdateBigEntityLogDocumentAnalysisNumber() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_2_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLog.setAnalysisNumber(entityLog.getAnalysisNumber() + 1);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getAnalysisNumber(), entityLog.getAnalysisNumber());
}
@Test
@SneakyThrows
public void testFindEntries() {
var latest = entityLogMongoService.findLatestAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID)
.orElseThrow();
var latestChangedEntries = entityLogMongoService.findAllEntityLogEntriesByAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, latest);
assertEquals(latestChangedEntries.size(), 490);
var manuallyChangedEntries = entityLogMongoService.findAllEntityLogEntriesWithManualChanges(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertEquals(manuallyChangedEntries.size(), 1013);
}
@Test
@SneakyThrows
public void testFindLatestAnalysisNumber() {
assertEquals(entityLogMongoService.findLatestAnalysisNumber(TEST_DOSSIER_2_ID, TEST_FILE_2_ID)
.orElse(-1), 8);
}
@Test
@SneakyThrows
public void testDeleteEntityLog() {
entityLogMongoService.deleteEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertFalse(entityLogMongoService.entityLogDocumentExists(TEST_DOSSIER_2_ID, TEST_FILE_2_ID));
assertEquals(entityLogMongoService.findAllEntriesByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID).size(), 0);
}
@Test
@SneakyThrows
public void testInsertBigEntityLogDocument() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_2_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_2_ID, TEST_FILE_2_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_2_ID, TEST_FILE_2_ID);
assertTrue(found.isPresent());
}
@Test
@SneakyThrows
public void testInsertEntityLogDocument() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
}
@Test
@SneakyThrows
public void testFindEntityLogDocumentById() {
Optional<EntityLog> entityLogDocumentById = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE_ID);
assert (entityLogDocumentById.isPresent());
}
//@Test
//@SneakyThrows
//public void testRedactionStorageService() {
//
// var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
// ObjectMapper objectMapper = new ObjectMapper();
// objectMapper.registerModule(new JavaTimeModule());
//
// EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
// EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
//
// String jsonData = objectMapper.writeValueAsString(entityLogDocument);
// InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes());
//
// minioInsert(jsonDataStream, jsonData.length());
//}
//
//
//private void minioInsert(InputStream inputStream, int length) {
//
// try {
// MinioClient minioClient = MinioClient.builder()
// .endpoint("http://localhost:9000")
// .credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ")
// .build();
//
// String contentType = "application/json";
//
// minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID)
// .stream(inputStream, length, -1).contentType(contentType).build());
//
// System.out.println("JSON file uploaded successfully!");
//
// } catch (Exception e) {
// e.printStackTrace();
// }
//}
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, LiquibaseAutoConfiguration.class, DataSourceAutoConfiguration.class})
@Import({MongoDbConfiguration.class, MultiTenantMongoDBFactory.class, MongoClientCache.class, MongoDataSources.class, MongoConnectionProviderImpl.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE)})
public static class MongoTestConfiguration {
}
@Slf4j
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
TestPropertyValues.of("spring.data.mongodb.auto-index-creation=true",
"MONGODB_HOST=" + HOSTNAME,
"MONGODB_PORT=" + PORT,
"MONGODB_USER=" + USERNAME,
"MONGODB_PASSWORD=" + PASSWORD,
"storage.bucket=name=redaction",
"storage.endpoint=http://localhost:9000",
"storage.key=L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya",
"storage.secret=Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ").applyTo(configurableApplicationContext.getEnvironment());
}
}
private final String NEW_ENTITY_LOG_ENTRY = """
{
"id": "05240998f2e657d739d59d220094702a",
"type": "CBI_author",
"entryType": "ENTITY",
"state": "REMOVED",
"value": "Amato",
"reason": "Author found",
"matchedRule": "CBI.0.0",
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
"imported": false,
"containingNodeId": [
1,
0
],
"closestHeadline": "",
"section": "[1, 0]: Paragraph: Almond R.H. Almond Richard",
"color": null,
"positions": [
{
"rectangle": [
56.8,
639.3,
32.59199,
12.642
],
"pageNumber": 2
}
],
"textBefore": "S. Amarasekare Amarasingham ",
"textAfter": " Amato S.L Amato",
"startOffset": 5137,
"endOffset": 5142,
"imageHasTransparency": false,
"dictionaryEntry": true,
"dossierDictionaryEntry": false,
"excluded": false,
"changes": [
{
"analysisNumber": 1,
"type": "ADDED",
"dateTime": "2024-03-13T08:44:54.811245839Z"
},
{
"analysisNumber": 3,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:11.339124572Z"
},
{
"analysisNumber": 4,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:19.694336707Z"
},
{
"analysisNumber": 5,
"type": "REMOVED",
"dateTime": "2024-03-13T08:48:21.670287664Z"
},
{
"analysisNumber": 6,
"type": "REMOVED",
"dateTime": "2024-03-13T08:58:41.082712591Z"
},
{
"analysisNumber": 7,
"type": "REMOVED",
"dateTime": "2024-03-13T08:59:31.444615299Z"
},
{
"analysisNumber": 8,
"type": "REMOVED",
"dateTime": "2024-03-13T09:05:22.006293189Z"
}
],
"manualChanges": [],
"engines": [
"DICTIONARY"
],
"reference": [],
"importedRedactionIntersections": [],
"numberOfComments": 0
}
""";
}

65652
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/6f2a9d4b5d4e11211fc0d7732fd45b78.ENTITY_LOG.json
View File

File diff suppressed because it is too large Load Diff

30537
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/b2cbdd4dca0aa1aa0ebbfc5cc1462df0.ENTITY_LOG.json
View File

File diff suppressed because it is too large Load Diff

120327
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/bigentityfileid.ENTITY_LOG.json
View File

File diff suppressed because it is too large Load Diff

7
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/mongo.changelog-tenant.xml
View File

@ -1,7 +0,0 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd">
<include file="/mongo/liquibase/changelog/tenant/1-initial-database.changelog.xml"/>
</databaseChangeLog>

224
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/1-initial-database.changelog-with-validation.xml
View File

@ -1,224 +0,0 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="createEntityLogEntryCollection" author="maverick">
<ext:createCollection collectionName="entity-log-entries">
<ext:options>
{
validator: {
$jsonSchema: {
bsonType: "object",
required: ["entryId", "entityLogId", "type", "entryType", "state", "value", "reason", "matchedRule", "legalBasis", "containingNodeId", "closestHeadline", "section",
"positions", "textBefore", "textAfter", "startOffset", "endOffset", "imageHasTransparency", "dictionaryEntry", "dossierDictionaryEntry", "excluded", "changes",
"manualChanges", "engines", "reference", "importedRedactionIntersections", "numberOfComments"],
properties: {
entryId: {
bsonType: "string",
description: "The Entry ID"
},
entityLogId: {
bsonType: "string",
description: "The Entity Log ID"
},
type: {
bsonType: "string",
description: "The Type"
},
entryType: {
bsonType: "string",
description: "The Entry Type"
},
state: {
bsonType: "string",
description: "The Entry State"
},
value: {
bsonType: "string",
description: "The Value"
},
reason: {
bsonType: "string",
description: "The Reason"
},
matchedRule: {
bsonType: "string",
description: "The Matched Rule"
},
legalBasis: {
bsonType: "string",
description: "The Legal Basis"
},
containingNodeId: {
bsonType: "array",
items: {
bsonType: "int",
description: "The Containing Node ID"
}
},
closestHeadline: {
bsonType: "string",
description: "The Closest Headline"
},
section: {
bsonType: "string",
description: "The Section"
},
positions: {
bsonType: "array",
description: "The Positions",
items: {
bsonType: "object"
}
},
textBefore: {
bsonType: "string",
description: "Text before the entry"
},
textAfter: {
bsonType: "string",
description: "Text after the entry"
},
startOffset: {
bsonType: "int",
description: "Start offset of the entry"
},
endOffset: {
bsonType: "int",
description: "End offset of the entry"
},
imageHasTransparency: {
bsonType: "bool",
description: "Whether the image has transparency"
},
dictionaryEntry: {
bsonType: "bool",
description: "Whether it's a dictionary entry"
},
dossierDictionaryEntry: {
bsonType: "bool",
description: "Whether it's a dossier dictionary entry"
},
excluded: {
bsonType: "bool",
description: "Whether it's excluded"
},
changes: {
bsonType: "array",
description: "The Changes",
items: {
bsonType: "object"
}
},
manualChanges: {
bsonType: "array",
description: "The Manual Changes",
items: {
bsonType: "object"
}
},
engines: {
bsonType: "array",
description: "The Engines",
items: {
bsonType: "string"
}
},
reference: {
bsonType: "array",
description: "The Reference",
items: {
bsonType: "string"
}
},
importedRedactionIntersections: {
bsonType: "array",
description: "The Imported Redaction Intersections",
items: {
bsonType: "string"
}
},
numberOfComments: {
bsonType: "int",
description: "The Number of Comments"
}
}
}
},
validationAction: "warn",
validationLevel: "strict"
}
</ext:options>
</ext:createCollection>
<ext:createCollection collectionName="entity-logs">
<ext:options>
{
validator: {
$jsonSchema: {
bsonType: "object",
required: ["dossierId", "fileId", "analysisVersion", "analysisNumber", "entityLogEntryDocument", "legalBasis"],
properties: {
dossierId: {
bsonType: "string",
description: "The Dossier ID"
},
fileId: {
bsonType: "string",
description: "The File ID"
},
analysisVersion: {
bsonType: "long",
description: "The Analysis Version"
},
analysisNumber: {
bsonType: "int",
description: "The Analysis Number"
},
entityLogEntryDocument: {
bsonType: "array",
description: "The Entity Log Entry Documents",
items: {
bsonType: "objectId"
}
},
legalBasis: {
bsonType: "array",
description: "The Legal Basis",
items: {
bsonType: "object"
}
},
dictionaryVersion: {
bsonType: "long",
description: "The Dictionary Version"
},
dossierDictionaryVersion: {
bsonType: "long",
description: "The Dossier Dictionary Version"
},
rulesVersion: {
bsonType: "long",
description: "The Rules Version"
},
legalBasisVersion: {
bsonType: "long",
description: "The Legal Basis Version"
}
}
}
},
validationAction: "warn",
validationLevel: "strict"
}
</ext:options>
</ext:createCollection>
</changeSet>
</databaseChangeLog>

17
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/1-initial-database.changelog.xml
View File

@ -1,17 +0,0 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="createEntityLogEntryCollection" author="maverick">
<ext:createCollection collectionName="entity-log-entries"/>
<ext:createCollection collectionName="entity-logs"/>
</changeSet>
</databaseChangeLog>

37
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/2-create-indices-for-entries.xml
View File

@ -1,37 +0,0 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="createEntityLogEntryCollection" author="maverick">
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"_id": 1,
"positions.pageNumber": 1
}
</ext:keys>
<ext:options>
{name: "positions_pageNumber_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"_id": 1,
"changes.analysisNumber": -1
}
</ext:keys>
<ext:options>
{name: "changes_analysisNumber_index"}
</ext:options>
</ext:createIndex>
</changeSet>
</databaseChangeLog>

26
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/example-remove-entry-number-of-comments.changelog.xml
View File

@ -1,26 +0,0 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="1" author="maverick">
<ext:runCommand>
<ext:command>
{
update: "entity-log-entries",
updates: [
{
q: {},
u: { $unset: { "numberOfComments": "" } },
multi: true
}
]
}
</ext:command>
</ext:runCommand>
</changeSet>
</databaseChangeLog>

120408
redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/upd_bigentityfileid.ENTITY_LOG.json
View File

File diff suppressed because it is too large Load Diff