diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 1a36bb7b..78decd58 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -18,6 +18,7 @@ val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" val persistenceServiceVersion = "maverick-mongo" val springBootStarterVersion = "3.1.5" +val testContainersVersion = "1.19.7" configurations { all { @@ -65,10 +66,6 @@ dependencies { implementation("org.reflections:reflections:0.10.2") - //todo: remove - implementation("io.minio:minio:8.5.9") - - implementation("org.liquibase:liquibase-core:4.24.0") implementation("org.liquibase.ext:liquibase-mongodb:4.24.0") @@ -77,6 +74,9 @@ dependencies { testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}") testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}") + testImplementation("org.testcontainers:testcontainers:${testContainersVersion}") + testImplementation("org.testcontainers:junit-jupiter:${testContainersVersion}") + testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}") testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}") testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") { @@ -87,6 +87,12 @@ dependencies { } } +dependencyManagement { + imports { + mavenBom("org.testcontainers:testcontainers-bom:${testContainersVersion}") + } +} + tasks.test { configure { excludes = listOf("org/drools/**/*") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java index 8a2dea3c..b3ed26c4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java @@ -6,7 +6,6 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import org.jetbrains.annotations.NotNull; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument; @@ -126,7 +125,6 @@ public class EntityLogMongoService { } - @NotNull private EntityLogDocument getEntityLogDocument(String entityLogId) { Optional optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 25adb2fc..698b9f80 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -22,16 +22,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; -import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; -import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index f37de123..4b44651d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import java.io.File; @@ -16,12 +17,20 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.bson.BsonArray; +import org.bson.BsonDocument; +import org.bson.BsonString; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.mockito.stubbing.Answer; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.core.io.ClassPathResource; +import org.springframework.test.context.ContextConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; @@ -38,19 +47,28 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer; import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantsClient; +import com.knecon.fforesight.tenantcommons.model.MongoDBConnection; +import com.mongodb.MongoCommandException; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +@ContextConfiguration(initializers = {AbstractRedactionIntegrationTest.Initializer.class}) public abstract class AbstractRedactionIntegrationTest { protected static final String VERTEBRATE_INDICATOR = "vertebrate"; @@ -129,6 +147,9 @@ public abstract class AbstractRedactionIntegrationTest { @MockBean private TenantsClient tenantsClient; + @MockBean + private MongoConnectionProvider mongoConnectionProvider; + protected final Map> dictionary = new HashMap<>(); protected final Map> dossierDictionary = new HashMap<>(); protected final Map> falsePositive = new HashMap<>(); @@ -552,4 +573,72 @@ public abstract class AbstractRedactionIntegrationTest { } + + // todo: 8702 check if this is the way + @BeforeEach + protected void mockProvideMongoDBConnection() { + + TenantContext.setTenantId("redaction"); + + var mongoInstance = MongoDBTestContainer.getInstance(); + + when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder() + .host(mongoInstance.getHost()) + .port(String.valueOf(mongoInstance.getFirstMappedPort())) + .database(MongoDBTestContainer.MONGO_DATABASE) + .username(MongoDBTestContainer.MONGO_USERNAME) + .password(MongoDBTestContainer.MONGO_PASSWORD) + .build()); + } + + + @Slf4j + static class Initializer implements ApplicationContextInitializer { + + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + + var mongoInstance = MongoDBTestContainer.getInstance(); + mongoInstance.start(); + createMongoDBDatabase(mongoInstance); + + log.info("Hosts are - MongoDB: {}", mongoInstance.getHost()); + + TestPropertyValues.of("MONGODB_HOST=" + mongoInstance.getHost(), + "MONGODB_PORT=" + mongoInstance.getFirstMappedPort(), + "MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME, + "MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD).applyTo(configurableApplicationContext.getEnvironment()); + + } + + } + + + private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) { + + try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/", + MongoDBTestContainer.MONGO_USERNAME, + MongoDBTestContainer.MONGO_PASSWORD, + mongoDBTestContainer.getHost(), + mongoDBTestContainer.getFirstMappedPort()))) { + MongoDatabase database = mongoClient.getDatabase(MongoDBTestContainer.MONGO_DATABASE); + BsonDocument createUserCommand = new BsonDocument(); + createUserCommand.append("createUser", new BsonString(MongoDBTestContainer.MONGO_USERNAME)); + createUserCommand.append("pwd", new BsonString(MongoDBTestContainer.MONGO_PASSWORD)); + BsonArray roles = new BsonArray(); + roles.add(new BsonString("readWrite")); + createUserCommand.append("roles", roles); + + try { + database.runCommand(createUserCommand); + } catch (MongoCommandException mongoCommandException) { + // ignore user already exists + if (mongoCommandException.getErrorCode() != 51003) { + throw mongoCommandException; + } + + } + + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java index b9fbe211..14d9a44a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java @@ -35,7 +35,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java index e710b2e4..b1db3cde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java @@ -10,7 +10,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.springframework.data.mongodb.core.MongoTemplate; -import com.knecon.fforesight.tenantcommons.model.TenantResponse; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java index 6d5e4fc4..552b4ecb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java @@ -6,8 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; -import java.io.ByteArrayInputStream; -import java.io.InputStream; import java.util.List; import java.util.Optional; @@ -37,7 +35,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; @@ -53,8 +50,6 @@ import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantsClient; import com.knecon.fforesight.tenantcommons.model.MongoDBConnection; -import io.minio.MinioClient; -import io.minio.PutObjectArgs; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -250,43 +245,43 @@ public class MyOtherMongoTest { } - @Test - @SneakyThrows - public void testRedactionStorageService() { - - var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID)); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new JavaTimeModule()); - - EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class); - EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog); - - String jsonData = objectMapper.writeValueAsString(entityLogDocument); - InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes()); - - minioInsert(jsonDataStream, jsonData.length()); - } - - - private void minioInsert(InputStream inputStream, int length) { - - try { - MinioClient minioClient = MinioClient.builder() - .endpoint("http://localhost:9000") - .credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ") - .build(); - - String contentType = "application/json"; - - minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID) - .stream(inputStream, length, -1).contentType(contentType).build()); - - System.out.println("JSON file uploaded successfully!"); - - } catch (Exception e) { - e.printStackTrace(); - } - } + //@Test + //@SneakyThrows + //public void testRedactionStorageService() { +// + // var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID)); + // ObjectMapper objectMapper = new ObjectMapper(); + // objectMapper.registerModule(new JavaTimeModule()); +// + // EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class); + // EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog); +// + // String jsonData = objectMapper.writeValueAsString(entityLogDocument); + // InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes()); +// + // minioInsert(jsonDataStream, jsonData.length()); + //} +// +// + //private void minioInsert(InputStream inputStream, int length) { +// + // try { + // MinioClient minioClient = MinioClient.builder() + // .endpoint("http://localhost:9000") + // .credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ") + // .build(); +// + // String contentType = "application/json"; +// + // minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID) + // .stream(inputStream, length, -1).contentType(contentType).build()); +// + // System.out.println("JSON file uploaded successfully!"); +// + // } catch (Exception e) { + // e.printStackTrace(); + // } + //} @Configuration diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java new file mode 100644 index 00000000..701c4eea --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java @@ -0,0 +1,35 @@ +package com.iqser.red.service.redaction.v1.server.testcontainers; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +public final class MongoDBTestContainer extends GenericContainer { + + private static final String IMAGE_VERSION = "mongo:7.0.2"; + public static final Integer MONGO_PORT = 27017; + public static final String MONGO_DATABASE = "mongo_database"; + public static final String MONGO_PASSWORD = "mongo_password"; + public static final String MONGO_USERNAME = "mongo_username"; + private static MongoDBTestContainer mongoDB; + + + private MongoDBTestContainer() { + + super(DockerImageName.parse(IMAGE_VERSION)); + + } + + + public static MongoDBTestContainer getInstance() { + + if (mongoDB == null) { + mongoDB = new MongoDBTestContainer().withEnv("MONGO_INITDB_ROOT_USERNAME", MONGO_USERNAME) + .withEnv("MONGO_INITDB_ROOT_PASSWORD", MONGO_PASSWORD) + .withEnv("MONGO_INITDB_DATABASE", MONGO_DATABASE) + .withExposedPorts(MONGO_PORT); + + } + return mongoDB; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml index df20094d..b16c3b9d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml @@ -44,6 +44,3 @@ management: health.enabled: true endpoints.web.exposure.include: prometheus, health, metrics metrics.export.prometheus.enabled: true - - -logging.level.root: debug