Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* added tests
This commit is contained in:
maverickstuder 2024-03-25 15:24:46 +01:00
parent 6ffffd75d9
commit 4eb321c1d1
6 changed files with 271557 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

238
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMongoServiceTest.java
View File

@ -1,6 +1,11 @@
package com.iqser.red.service.persistence.server.integration.tests;
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import java.util.Optional;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
@ -10,32 +15,245 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.mongo.EntityLogMongoService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
public class EntityLogMongoServiceTest extends AbstractPersistenceServerServiceTest {
@Autowired
private EntityLogMongoService entityLogMongoService;
private final String ENTITY_LOG = "sample.ENTITY_LOG.json";
private final String ENTITY_LOG1 = "files/entity-log/b2cbdd4dca0aa1aa0ebbfc5cc1462df0.ENTITY_LOG.json";
private final String ENTITY_LOG2 = "files/entity-log/6f2a9d4b5d4e11211fc0d7732fd45b78.ENTITY_LOG.json";
private final String ENTITY_LOG3_BEFORE = "files/entity-log/c3b23116f2d277170d303bfc6fb82e6a-before.ENTITY_LOG.json";
private final String ENTITY_LOG3_AFTER = "files/entity-log/c3b23116f2d277170d303bfc6fb82e6a-after.ENTITY_LOG.json";
private static final String TEST_DOSSIER_ID = "91ce8e90-9aec-473c-b8c3-cbe16443ad34";
private static final String TEST_FILE1_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
private static final String TEST_FILE2_ID = "6f2a9d4b5d4e11211fc0d7732fd45b78";
private static final String TEST_FILE3_ID = "c3b23116f2d277170d303bfc6fb82e6a";
private static final String TEST_DOSSIER_ID = "6f2a9d4b5d4e11211fc0d7732fd45b78";
private static final String TEST_FILE_ID = "b2cbdd4dca0aa1aa0ebbfc5cc1462df0";
@Test
@SneakyThrows
@Disabled
public void testInsertEntityLogDocument() {
public void testMultiTenantInserts() {
var file = new ClassPathResource(String.format(ENTITY_LOG));
var file = new ClassPathResource(String.format(ENTITY_LOG1));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLogToStore = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE1_ID, entityLogToStore);
var optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE1_ID);
assertTrue(optionalEntityLog.isPresent());
EntityLog entityLogFromStorage = optionalEntityLog.get();
assertEquals(entityLogFromStorage, entityLogToStore);
TenantContext.setTenantId("redaction2");
optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE1_ID);
assertTrue(optionalEntityLog.isEmpty());
file = new ClassPathResource(String.format(ENTITY_LOG2));
entityLogToStore = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE2_ID, entityLogToStore);
optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE2_ID);
assertTrue(optionalEntityLog.isPresent());
entityLogFromStorage = optionalEntityLog.get();
assertEquals(entityLogFromStorage, entityLogToStore);
TenantContext.setTenantId("redaction");
optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE2_ID);
assertTrue(optionalEntityLog.isEmpty());
}
@Test
@SneakyThrows
public void testInsertAndDeleteEntityLogDocument() {
var file = new ClassPathResource(String.format(ENTITY_LOG1));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLogToStore = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE1_ID, entityLogToStore);
var optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE1_ID);
assertTrue(optionalEntityLog.isPresent());
EntityLog entityLogFromStorage = optionalEntityLog.get();
assertEquals(entityLogFromStorage, entityLogToStore);
entityLogMongoService.deleteEntityLog(TEST_DOSSIER_ID, TEST_FILE1_ID);
optionalEntityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE1_ID);
assertTrue(optionalEntityLog.isEmpty());
}
@Test
@SneakyThrows
public void testFindAndUpdateEntityLogDocument() {
var file = new ClassPathResource(ENTITY_LOG3_BEFORE);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE3_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE3_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getEntityLogEntry().size(), 1706);
assertEquals(entityLogMongoService.findLatestAnalysisNumber(TEST_DOSSIER_ID, TEST_FILE3_ID)
.orElse(-1), 8);
var latest = entityLogMongoService.findLatestAnalysisNumber(TEST_DOSSIER_ID, TEST_FILE3_ID)
.orElseThrow();
var latestChangedEntries = entityLogMongoService.findAllEntityLogEntriesByAnalysisNumber(TEST_DOSSIER_ID, TEST_FILE3_ID, latest);
assertEquals(latestChangedEntries.size(), 490);
var manuallyChangedEntries = entityLogMongoService.findAllEntityLogEntriesWithManualChanges(TEST_DOSSIER_ID, TEST_FILE3_ID);
assertEquals(manuallyChangedEntries.size(), 1014);
String NEW_ENTITY_LOG_ENTRY = """
{
"id": "05240998f2e657d739d59d220094702a",
"type": "CBI_author",
"entryType": "ENTITY",
"state": "REMOVED",
"value": "Amato",
"reason": "Author found",
"matchedRule": "CBI.0.0",
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
"imported": false,
"containingNodeId": [
1,
0
],
"closestHeadline": "",
"section": "[1, 0]: Paragraph: Almond R.H. Almond Richard",
"color": null,
"positions": [
{
"rectangle": [
56.8,
639.3,
32.59199,
12.642
],
"pageNumber": 2
}
],
"textBefore": "S. Amarasekare Amarasingham ",
"textAfter": " Amato S.L Amato",
"startOffset": 5137,
"endOffset": 5142,
"imageHasTransparency": false,
"dictionaryEntry": true,
"dossierDictionaryEntry": false,
"excluded": false,
"changes": [
{
"analysisNumber": 1,
"type": "ADDED",
"dateTime": "2024-03-13T08:44:54.811245839Z"
},
{
"analysisNumber": 3,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:11.339124572Z"
},
{
"analysisNumber": 4,
"type": "REMOVED",
"dateTime": "2024-03-13T08:47:19.694336707Z"
},
{
"analysisNumber": 5,
"type": "REMOVED",
"dateTime": "2024-03-13T08:48:21.670287664Z"
},
{
"analysisNumber": 6,
"type": "REMOVED",
"dateTime": "2024-03-13T08:58:41.082712591Z"
},
{
"analysisNumber": 7,
"type": "REMOVED",
"dateTime": "2024-03-13T08:59:31.444615299Z"
},
{
"analysisNumber": 8,
"type": "REMOVED",
"dateTime": "2024-03-13T09:05:22.006293189Z"
}
],
"manualChanges": [],
"engines": [
"DICTIONARY"
],
"reference": [],
"importedRedactionIntersections": [],
"numberOfComments": 0
}
""";
EntityLogEntry entityLogEntry = objectMapper.readValue(NEW_ENTITY_LOG_ENTRY, EntityLogEntry.class);
entityLogMongoService.insertEntityLogEntries(TEST_DOSSIER_ID, TEST_FILE3_ID, List.of(entityLogEntry));
found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE3_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getEntityLogEntry().size(), 1707);
}
@Test
@SneakyThrows
public void testUpdateEntityLogDocumentByOverride() {
var file = new ClassPathResource(ENTITY_LOG3_BEFORE);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLogMongoService.insertEntityLog(TEST_DOSSIER_ID, TEST_FILE3_ID, entityLog);
Optional<EntityLog> found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE3_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getEntityLogEntry().size(), 1706);
file = new ClassPathResource(ENTITY_LOG3_AFTER);
entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
entityLog.setAnalysisNumber(entityLog.getAnalysisNumber() + 1);
entityLogMongoService.saveEntityLog(TEST_DOSSIER_ID, TEST_FILE3_ID, entityLog);
found = entityLogMongoService.findEntityLogByDossierIdAndFileId(TEST_DOSSIER_ID, TEST_FILE3_ID);
assertTrue(found.isPresent());
assertEquals(found.get().getEntityLogEntry().size(), 1707);
assertEquals(found.get().getAnalysisNumber(), 9);
}
}

67
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -235,11 +235,11 @@ public abstract class AbstractPersistenceServerServiceTest {
@MockBean
private UsersClient usersClient;
@Autowired
private EncryptionDecryptionService encryptionDecryptionService;
protected EncryptionDecryptionService encryptionDecryptionService;
@Autowired
private TenantCreatedListener tenantCreatedListener;
protected TenantCreatedListener tenantCreatedListener;
@Autowired
private MongoTenantCreatedListener mongoTenantCreatedListener;
protected MongoTenantCreatedListener mongoTenantCreatedListener;
private static String[] getAllRoles() {
@ -273,7 +273,7 @@ public abstract class AbstractPersistenceServerServiceTest {
@BeforeEach
public void setupOptimize() {
createDefaultTenant();
createTenants();
TenantContext.setTenantId("redaction");
@ -322,7 +322,7 @@ public abstract class AbstractPersistenceServerServiceTest {
}
private void createDefaultTenant() {
private void createTenants() {
var postgreSQLContainerMaster = SpringPostgreSQLTestContainer.getInstance().withDatabaseName("integration-tests-db-master").withUsername("sa").withPassword("sa");
var mongoDbContainer = MongoDBTestContainer.getInstance();
@ -372,12 +372,58 @@ public abstract class AbstractPersistenceServerServiceTest {
.database(MONGO_DATABASE)
.build());
var redactionTenant2 = new TenantResponse();
redactionTenant2.setTenantId("redaction2");
redactionTenant2.setGuid("redaction2");
redactionTenant2.setDisplayName("redaction2");
redactionTenant2.setAuthDetails(new AuthDetails());
redactionTenant2.setDatabaseConnection(DatabaseConnection.builder()
.driver("postgresql")
.host(postgreSQLContainerMaster.getHost())
.port(port)
.database("integration-tests-db-master")
.schema("public")
.username("sa")
.password(encryptionDecryptionService.encrypt("sa"))
.build());
redactionTenant2.setSearchConnection(SearchConnection.builder()
.hosts(Set.of("elasticsearchHost"))
.port(9200)
.scheme("https")
.username("elastic")
.numberOfShards("1")
.numberOfReplicas("5")
.build());
redactionTenant2.setS3StorageConnection(S3StorageConnection.builder()
.key("key")
.secret("secret")
.signerType("signerType")
.bucketName("bucketName")
.region("eu")
.endpoint("endpoint")
.build());
redactionTenant2.setMongoDBConnection(MongoDBConnection.builder()
.host(mongoDbContainer.getHost())
.port(String.valueOf(mongoDbContainer.getFirstMappedPort()))
.username(MONGO_USERNAME)
.password(encryptionDecryptionService.encrypt(MONGO_PASSWORD))
.database("redaction2")
.build());
when(tenantsClient.getTenant("redaction")).thenReturn(redactionTenant);
when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant));
when(tenantsClient.getTenant("redaction2")).thenReturn(redactionTenant2);
when(tenantsClient.getTenants()).thenReturn(List.of(redactionTenant, redactionTenant2));
try {
tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction"));
mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction"));
tenantCreatedListener.createTenant(new TenantCreatedEvent("redaction2"));
mongoTenantCreatedListener.createTenant(new MongoTenantCreatedEvent("redaction2"));
} catch (Exception e) {
e.printStackTrace();
@ -475,7 +521,8 @@ public abstract class AbstractPersistenceServerServiceTest {
var mongoInstance = MongoDBTestContainer.getInstance();
mongoInstance.start();
createMongoDBDatabase(mongoInstance);
createMongoDBDatabase(mongoInstance, "redaction");
createMongoDBDatabase(mongoInstance, "redaction2");
log.info("Hosts are - Redis: {}, Postgres: {}, MongoDB: {}", redisContainer.getHost(), postgreSQLContainerMaster.getHost(), mongoInstance.getHost());
@ -493,19 +540,19 @@ public abstract class AbstractPersistenceServerServiceTest {
}
private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) {
private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer, String databaseName) {
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
MONGO_USERNAME,
MONGO_PASSWORD,
mongoDBTestContainer.getHost(),
mongoDBTestContainer.getFirstMappedPort()))) {
MongoDatabase database = mongoClient.getDatabase(MONGO_DATABASE);
MongoDatabase database = mongoClient.getDatabase(databaseName);
BsonDocument createUserCommand = new BsonDocument();
createUserCommand.append("createUser", new BsonString(MONGO_USERNAME));
createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD));
BsonArray roles = new BsonArray();
roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(MONGO_DATABASE)));
roles.add(new BsonDocument("role", new BsonString("dbOwner")).append("db", new BsonString(databaseName)));
createUserCommand.append("roles", roles);
try {

0
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/sample.ENTITY_LOG.json → persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/6f2a9d4b5d4e11211fc0d7732fd45b78.ENTITY_LOG.json
View File

30537
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/b2cbdd4dca0aa1aa0ebbfc5cc1462df0.ENTITY_LOG.json Normal file
View File

File diff suppressed because it is too large Load Diff

120408
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/c3b23116f2d277170d303bfc6fb82e6a-after.ENTITY_LOG.json Normal file
View File

File diff suppressed because it is too large Load Diff

120327
persistence-service-v1/persistence-service-server-v1/src/test/resources/files/entity-log/c3b23116f2d277170d303bfc6fb82e6a-before.ENTITY_LOG.json Normal file
View File

File diff suppressed because it is too large Load Diff