Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source 
* pre-test eval version
This commit is contained in:
maverickstuder 2024-03-19 17:00:44 +01:00
parent 448eed1b29
commit f09fea4943
9 changed files with 171 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -18,6 +18,7 @@ val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "maverick-mongo"
val springBootStarterVersion = "3.1.5"
val testContainersVersion = "1.19.7"
configurations {
all {
@ -65,10 +66,6 @@ dependencies {
implementation("org.reflections:reflections:0.10.2")
//todo: remove
implementation("io.minio:minio:8.5.9")
implementation("org.liquibase:liquibase-core:4.24.0")
implementation("org.liquibase.ext:liquibase-mongodb:4.24.0")
@ -77,6 +74,9 @@ dependencies {
testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
testImplementation("org.testcontainers:testcontainers:${testContainersVersion}")
testImplementation("org.testcontainers:junit-jupiter:${testContainersVersion}")
testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
@ -87,6 +87,12 @@ dependencies {
}
}
dependencyManagement {
imports {
mavenBom("org.testcontainers:testcontainers-bom:${testContainersVersion}")
}
}
tasks.test {
configure<JacocoTaskExtension> {
excludes = listOf("org/drools/**/*")

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java
View File

@ -6,7 +6,6 @@ import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
@ -126,7 +125,6 @@ public class EntityLogMongoService {
}
@NotNull
private EntityLogDocument getEntityLogDocument(String entityLogId) {
Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(entityLogId);

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -22,16 +22,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;

89
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.io.File;
@ -16,12 +17,20 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.bson.BsonArray;
import org.bson.BsonDocument;
import org.bson.BsonString;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.mockito.stubbing.Answer;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.ContextConfiguration;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
@ -38,19 +47,28 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer;
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import com.mongodb.MongoCommandException;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoDatabase;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ContextConfiguration(initializers = {AbstractRedactionIntegrationTest.Initializer.class})
public abstract class AbstractRedactionIntegrationTest {
protected static final String VERTEBRATE_INDICATOR = "vertebrate";
@ -129,6 +147,9 @@ public abstract class AbstractRedactionIntegrationTest {
@MockBean
private TenantsClient tenantsClient;
@MockBean
private MongoConnectionProvider mongoConnectionProvider;
protected final Map<String, List<String>> dictionary = new HashMap<>();
protected final Map<String, List<String>> dossierDictionary = new HashMap<>();
protected final Map<String, List<String>> falsePositive = new HashMap<>();
@ -552,4 +573,72 @@ public abstract class AbstractRedactionIntegrationTest {
}
// todo: 8702 check if this is the way
@BeforeEach
protected void mockProvideMongoDBConnection() {
TenantContext.setTenantId("redaction");
var mongoInstance = MongoDBTestContainer.getInstance();
when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder()
.host(mongoInstance.getHost())
.port(String.valueOf(mongoInstance.getFirstMappedPort()))
.database(MongoDBTestContainer.MONGO_DATABASE)
.username(MongoDBTestContainer.MONGO_USERNAME)
.password(MongoDBTestContainer.MONGO_PASSWORD)
.build());
}
@Slf4j
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
var mongoInstance = MongoDBTestContainer.getInstance();
mongoInstance.start();
createMongoDBDatabase(mongoInstance);
log.info("Hosts are - MongoDB: {}", mongoInstance.getHost());
TestPropertyValues.of("MONGODB_HOST=" + mongoInstance.getHost(),
"MONGODB_PORT=" + mongoInstance.getFirstMappedPort(),
"MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME,
"MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD).applyTo(configurableApplicationContext.getEnvironment());
}
}
private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) {
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
MongoDBTestContainer.MONGO_USERNAME,
MongoDBTestContainer.MONGO_PASSWORD,
mongoDBTestContainer.getHost(),
mongoDBTestContainer.getFirstMappedPort()))) {
MongoDatabase database = mongoClient.getDatabase(MongoDBTestContainer.MONGO_DATABASE);
BsonDocument createUserCommand = new BsonDocument();
createUserCommand.append("createUser", new BsonString(MongoDBTestContainer.MONGO_USERNAME));
createUserCommand.append("pwd", new BsonString(MongoDBTestContainer.MONGO_PASSWORD));
BsonArray roles = new BsonArray();
roles.add(new BsonString("readWrite"));
createUserCommand.append("roles", roles);
try {
database.runCommand(createUserCommand);
} catch (MongoCommandException mongoCommandException) {
// ignore user already exists
if (mongoCommandException.getErrorCode() != 51003) {
throw mongoCommandException;
}
}
}
}
}

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java
View File

@ -35,7 +35,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoTest.java
View File

@ -10,7 +10,6 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.data.mongodb.core.MongoTemplate;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;

79
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyOtherMongoTest.java
View File

@ -6,8 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
@ -37,7 +35,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.persistence.management.v1.processor.document.EntityLogDocument;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@ -53,8 +50,6 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -250,43 +245,43 @@ public class MyOtherMongoTest {
}
@Test
@SneakyThrows
public void testRedactionStorageService() {
var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
String jsonData = objectMapper.writeValueAsString(entityLogDocument);
InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes());
minioInsert(jsonDataStream, jsonData.length());
}
private void minioInsert(InputStream inputStream, int length) {
try {
MinioClient minioClient = MinioClient.builder()
.endpoint("http://localhost:9000")
.credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ")
.build();
String contentType = "application/json";
minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID)
.stream(inputStream, length, -1).contentType(contentType).build());
System.out.println("JSON file uploaded successfully!");
} catch (Exception e) {
e.printStackTrace();
}
}
//@Test
//@SneakyThrows
//public void testRedactionStorageService() {
//
// var file = new ClassPathResource(String.format("mongo/%s.ENTITY_LOG.json", TEST_FILE_ID));
// ObjectMapper objectMapper = new ObjectMapper();
// objectMapper.registerModule(new JavaTimeModule());
//
// EntityLog entityLog = objectMapper.readValue(file.getInputStream(), EntityLog.class);
// EntityLogDocument entityLogDocument = new EntityLogDocument(TEST_DOSSIER_ID, TEST_FILE_ID, entityLog);
//
// String jsonData = objectMapper.writeValueAsString(entityLogDocument);
// InputStream jsonDataStream = new ByteArrayInputStream(jsonData.getBytes());
//
// minioInsert(jsonDataStream, jsonData.length());
//}
//
//
//private void minioInsert(InputStream inputStream, int length) {
//
// try {
// MinioClient minioClient = MinioClient.builder()
// .endpoint("http://localhost:9000")
// .credentials("L2U4tR5Beaqh6UHPwvtDKTJSxWBXIiya", "Spbv4vleaFnIrSiLLG3AhbdQk6DMraBQ")
// .build();
//
// String contentType = "application/json";
//
// minioClient.putObject(PutObjectArgs.builder().bucket(DATABASE).object(TEST_FILE_ID)
// .stream(inputStream, length, -1).contentType(contentType).build());
//
// System.out.println("JSON file uploaded successfully!");
//
// } catch (Exception e) {
// e.printStackTrace();
// }
//}
@Configuration

35
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java Normal file
View File

@ -0,0 +1,35 @@
package com.iqser.red.service.redaction.v1.server.testcontainers;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.utility.DockerImageName;
public final class MongoDBTestContainer extends GenericContainer<MongoDBTestContainer> {
private static final String IMAGE_VERSION = "mongo:7.0.2";
public static final Integer MONGO_PORT = 27017;
public static final String MONGO_DATABASE = "mongo_database";
public static final String MONGO_PASSWORD = "mongo_password";
public static final String MONGO_USERNAME = "mongo_username";
private static MongoDBTestContainer mongoDB;
private MongoDBTestContainer() {
super(DockerImageName.parse(IMAGE_VERSION));
}
public static MongoDBTestContainer getInstance() {
if (mongoDB == null) {
mongoDB = new MongoDBTestContainer().withEnv("MONGO_INITDB_ROOT_USERNAME", MONGO_USERNAME)
.withEnv("MONGO_INITDB_ROOT_PASSWORD", MONGO_PASSWORD)
.withEnv("MONGO_INITDB_DATABASE", MONGO_DATABASE)
.withExposedPorts(MONGO_PORT);
}
return mongoDB;
}
}

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml
View File

@ -44,6 +44,3 @@ management:
health.enabled: true
endpoints.web.exposure.include: prometheus, health, metrics
metrics.export.prometheus.enabled: true
logging.level.root: debug