RED-8702: Explore document databases to store entityLog

* pre-test eval version

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -18,6 +18,7 @@ val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
 val persistenceServiceVersion = "maverick-mongo"
 val springBootStarterVersion = "3.1.5"
+val testContainersVersion = "1.19.7"
 
 configurations {
     all {
@@ -65,10 +66,6 @@ dependencies {
 
     implementation("org.reflections:reflections:0.10.2")
 
-    //todo: remove
-    implementation("io.minio:minio:8.5.9")
-
-
     implementation("org.liquibase:liquibase-core:4.24.0")
 
     implementation("org.liquibase.ext:liquibase-mongodb:4.24.0")
@@ -77,6 +74,9 @@ dependencies {
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
+    testImplementation("org.testcontainers:testcontainers:${testContainersVersion}")
+    testImplementation("org.testcontainers:junit-jupiter:${testContainersVersion}")
+
     testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
     testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
@@ -87,6 +87,12 @@ dependencies {
     }
 }
 
+dependencyManagement {
+    imports {
+        mavenBom("org.testcontainers:testcontainers-bom:${testContainersVersion}")
+    }
+}
+
 tasks.test {
     configure<JacocoTaskExtension> {
         excludes = listOf("org/drools/**/*")

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java

@@ -6,7 +6,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
@@ -126,7 +125,6 @@ public class EntityLogMongoService {
     }
 
 
-    @NotNull
     private EntityLogDocument getEntityLogDocument(String entityLogId) {
 
         Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -22,16 +22,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 
 import java.io.File;
@@ -16,12 +17,20 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.bson.BsonArray;
+import org.bson.BsonDocument;
+import org.bson.BsonString;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.mockito.stubbing.Answer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.boot.test.util.TestPropertyValues;
+import org.springframework.context.ApplicationContextInitializer;
+import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.ContextConfiguration;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
@@ -38,19 +47,28 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer;
 import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
 import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
+import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
+import com.mongodb.MongoCommandException;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoDatabase;
 
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
+@ContextConfiguration(initializers = {AbstractRedactionIntegrationTest.Initializer.class})
 public abstract class AbstractRedactionIntegrationTest {
 
     protected static final String VERTEBRATE_INDICATOR = "vertebrate";
@@ -129,6 +147,9 @@ public abstract class AbstractRedactionIntegrationTest {
     @MockBean
     private TenantsClient tenantsClient;
 
+    @MockBean
+    private MongoConnectionProvider mongoConnectionProvider;
+
     protected final Map<String, List<String>> dictionary = new HashMap<>();
     protected final Map<String, List<String>> dossierDictionary = new HashMap<>();
     protected final Map<String, List<String>> falsePositive = new HashMap<>();
@@ -552,4 +573,72 @@ public abstract class AbstractRedactionIntegrationTest {
 
     }
 
+
+    // todo: 8702 check if this is the way
+    @BeforeEach
+    protected void mockProvideMongoDBConnection() {
+
+        TenantContext.setTenantId("redaction");
+
+        var mongoInstance = MongoDBTestContainer.getInstance();
+
+        when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder()
+                                                                                     .host(mongoInstance.getHost())
+                                                                                     .port(String.valueOf(mongoInstance.getFirstMappedPort()))
+                                                                                     .database(MongoDBTestContainer.MONGO_DATABASE)
+                                                                                     .username(MongoDBTestContainer.MONGO_USERNAME)
+                                                                                     .password(MongoDBTestContainer.MONGO_PASSWORD)
+                                                                                     .build());
+    }
+
+
+    @Slf4j
+    static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
+
+        public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
+
+            var mongoInstance = MongoDBTestContainer.getInstance();
+            mongoInstance.start();
+            createMongoDBDatabase(mongoInstance);
+
+            log.info("Hosts are -  MongoDB: {}", mongoInstance.getHost());
+
+            TestPropertyValues.of("MONGODB_HOST=" + mongoInstance.getHost(),
+                                  "MONGODB_PORT=" + mongoInstance.getFirstMappedPort(),
+                                  "MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME,
+                                  "MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD).applyTo(configurableApplicationContext.getEnvironment());
+
+        }
+
+    }
+
+
+    private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) {
+
+        try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
+                                                                         MongoDBTestContainer.MONGO_USERNAME,
+                                                                         MongoDBTestContainer.MONGO_PASSWORD,
+                                                                         mongoDBTestContainer.getHost(),
+                                                                         mongoDBTestContainer.getFirstMappedPort()))) {
+            MongoDatabase database = mongoClient.getDatabase(MongoDBTestContainer.MONGO_DATABASE);
+            BsonDocument createUserCommand = new BsonDocument();
+            createUserCommand.append("createUser", new BsonString(MongoDBTestContainer.MONGO_USERNAME));
+            createUserCommand.append("pwd", new BsonString(MongoDBTestContainer.MONGO_PASSWORD));
+            BsonArray roles = new BsonArray();
+            roles.add(new BsonString("readWrite"));
+            createUserCommand.append("roles", roles);
+
+            try {
+                database.runCommand(createUserCommand);
+            } catch (MongoCommandException mongoCommandException) {
+                // ignore user already exists
+                if (mongoCommandException.getErrorCode() != 51003) {
+                    throw mongoCommandException;
+                }
+
+            }
+
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -35,7 +35,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java

@@ -10,7 +10,6 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.data.mongodb.core.MongoTemplate;
 
-import com.knecon.fforesight.tenantcommons.model.TenantResponse;
 import com.mongodb.client.MongoClient;
 import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoCollection;

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java

@@ -6,8 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
 import java.util.List;
 import java.util.Optional;
 
@@ -37,7 +35,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@@ -53,8 +50,6 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
 
-import io.minio.MinioClient;
-import io.minio.PutObjectArgs;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
@@ -250,43 +245,43 @@ public class MyOtherMongoTest {
     }
 
 
-    @Test
+    //@Test
-    @SneakyThrows
+    //@SneakyThrows
-    public void testRedactionStorageService() {
+    //public void testRedactionStorageService() {
-
+//
-        var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
+    //    var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
-        ObjectMapper objectMapper = new ObjectMapper();
+    //    ObjectMapper objectMapper = new ObjectMapper();
-        objectMapper.registerModule(new JavaTimeModule());
+    //    objectMapper.registerModule(new JavaTimeModule());
-
+//
-        EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
+    //    EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
-        EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
+    //    EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
-
+//
-        String jsonData = objectMapper.writeValueAsString(entityLogDocument);
+    //    String jsonData = objectMapper.writeValueAsString(entityLogDocument);
-        InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes());
+    //    InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes());
-
+//
-        minioInsert(jsonDataStream, jsonData.length());
+    //    minioInsert(jsonDataStream, jsonData.length());
-    }
+    //}
-
+//
-
+//
-    private void minioInsert(InputStream inputStream, int length) {
+    //private void minioInsert(InputStream inputStream, int length) {
-
+//
-        try {
+    //    try {
-            MinioClient minioClient = MinioClient.builder()
+    //        MinioClient minioClient = MinioClient.builder()
-                    .endpoint("http://localhost:9000")
+    //                .endpoint("http://localhost:9000")
-                    .credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ")
+    //                .credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ")
-                    .build();
+    //                .build();
-
+//
-            String contentType = "application/json";
+    //        String contentType = "application/json";
-
+//
-            minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID)
+    //        minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID)
-                                          .stream(inputStream, length, -1).contentType(contentType).build());
+    //                                      .stream(inputStream, length, -1).contentType(contentType).build());
-
+//
-            System.out.println("JSON file uploaded successfully!");
+    //        System.out.println("JSON file uploaded successfully!");
-
+//
-        } catch (Exception e) {
+    //    } catch (Exception e) {
-            e.printStackTrace();
+    //        e.printStackTrace();
-        }
+    //    }
-    }
+    //}
 
 
     @Configuration

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.redaction.v1.server.testcontainers;
+
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
+
+public final class MongoDBTestContainer extends GenericContainer<MongoDBTestContainer> {
+
+    private static final String IMAGE_VERSION = "mongo:7.0.2";
+    public static final Integer MONGO_PORT = 27017;
+    public static final String MONGO_DATABASE = "mongo_database";
+    public static final String MONGO_PASSWORD = "mongo_password";
+    public static final String MONGO_USERNAME = "mongo_username";
+    private static MongoDBTestContainer mongoDB;
+
+
+    private MongoDBTestContainer() {
+
+        super(DockerImageName.parse(IMAGE_VERSION));
+
+    }
+
+
+    public static MongoDBTestContainer getInstance() {
+
+        if (mongoDB == null) {
+            mongoDB = new MongoDBTestContainer().withEnv("MONGO_INITDB_ROOT_USERNAME", MONGO_USERNAME)
+                    .withEnv("MONGO_INITDB_ROOT_PASSWORD", MONGO_PASSWORD)
+                    .withEnv("MONGO_INITDB_DATABASE", MONGO_DATABASE)
+                    .withExposedPorts(MONGO_PORT);
+
+        }
+        return mongoDB;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml

@@ -44,6 +44,3 @@ management:
     health.enabled: true
   endpoints.web.exposure.include: prometheus, health, metrics
   metrics.export.prometheus.enabled: true
-
-
-logging.level.root: debug