diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 774dc096..66e76ee1 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -7,7 +7,7 @@ description = "redaction-service-api-v1" dependencies { implementation("org.springframework:spring-web:6.0.12") - implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.383.0") + implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.393.0") } publishing { diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 964b83de..2af3b4f6 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.107.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.383.0" +val persistenceServiceVersion = "2.393.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" @@ -41,7 +41,7 @@ dependencies { implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:storage-commons:2.45.0") - implementation("com.knecon.fforesight:tenant-commons:0.23.0") + implementation("com.knecon.fforesight:tenant-commons:0.24.0") implementation("com.knecon.fforesight:tracing-commons:0.5.0") implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}") diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java index e5b10742..09ba88b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java @@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongo import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration; +import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase; import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration; import io.micrometer.core.aop.TimedAspect; @@ -34,6 +35,7 @@ import io.micrometer.observation.aop.ObservedAspect; @EnableFeignClients(basePackageClasses = RulesClient.class) @EnableConfigurationProperties(RedactionServiceSettings.class) @EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence") +@EnableMongoLiquibase @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) public class Application { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml index 4df10cea..e0cc6d9f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml @@ -20,3 +20,9 @@ redaction-service: application: type: "RedactManager" + +multitenancy: + tenant: + mongo: + liquibase: + changeLog: classpath:mongo/changelog/mongo.changelog-tenant.xml \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml index 6078cc9f..0f9a07a7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml @@ -75,3 +75,10 @@ storage: application: type: "RedactManager" + + +multitenancy: + tenant: + mongo: + liquibase: + changeLog: classpath:mongo/changelog/mongo.changelog-tenant.xml diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml new file mode 100644 index 00000000..0ed1f074 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml @@ -0,0 +1,7 @@ + + + + diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog-with-validation.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog-with-validation.xml new file mode 100644 index 00000000..c988f71a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog-with-validation.xml @@ -0,0 +1,224 @@ + + + + + + + { + validator: { + $jsonSchema: { + bsonType: "object", + required: ["entryId", "entityLogId", "type", "entryType", "state", "value", "reason", "matchedRule", "legalBasis", "containingNodeId", "closestHeadline", "section", + "positions", "textBefore", "textAfter", "startOffset", "endOffset", "imageHasTransparency", "dictionaryEntry", "dossierDictionaryEntry", "excluded", "changes", + "manualChanges", "engines", "reference", "importedRedactionIntersections", "numberOfComments"], + properties: { + entryId: { + bsonType: "string", + description: "The Entry ID" + }, + entityLogId: { + bsonType: "string", + description: "The Entity Log ID" + }, + type: { + bsonType: "string", + description: "The Type" + }, + entryType: { + bsonType: "string", + description: "The Entry Type" + }, + state: { + bsonType: "string", + description: "The Entry State" + }, + value: { + bsonType: "string", + description: "The Value" + }, + reason: { + bsonType: "string", + description: "The Reason" + }, + matchedRule: { + bsonType: "string", + description: "The Matched Rule" + }, + legalBasis: { + bsonType: "string", + description: "The Legal Basis" + }, + containingNodeId: { + bsonType: "array", + items: { + bsonType: "int", + description: "The Containing Node ID" + } + }, + closestHeadline: { + bsonType: "string", + description: "The Closest Headline" + }, + section: { + bsonType: "string", + description: "The Section" + }, + positions: { + bsonType: "array", + description: "The Positions", + items: { + bsonType: "object" + } + }, + textBefore: { + bsonType: "string", + description: "Text before the entry" + }, + textAfter: { + bsonType: "string", + description: "Text after the entry" + }, + startOffset: { + bsonType: "int", + description: "Start offset of the entry" + }, + endOffset: { + bsonType: "int", + description: "End offset of the entry" + }, + imageHasTransparency: { + bsonType: "bool", + description: "Whether the image has transparency" + }, + dictionaryEntry: { + bsonType: "bool", + description: "Whether it's a dictionary entry" + }, + dossierDictionaryEntry: { + bsonType: "bool", + description: "Whether it's a dossier dictionary entry" + }, + excluded: { + bsonType: "bool", + description: "Whether it's excluded" + }, + changes: { + bsonType: "array", + description: "The Changes", + items: { + bsonType: "object" + } + }, + manualChanges: { + bsonType: "array", + description: "The Manual Changes", + items: { + bsonType: "object" + } + }, + engines: { + bsonType: "array", + description: "The Engines", + items: { + bsonType: "string" + } + }, + reference: { + bsonType: "array", + description: "The Reference", + items: { + bsonType: "string" + } + }, + importedRedactionIntersections: { + bsonType: "array", + description: "The Imported Redaction Intersections", + items: { + bsonType: "string" + } + }, + numberOfComments: { + bsonType: "int", + description: "The Number of Comments" + } + } + } + }, + validationAction: "warn", + validationLevel: "strict" + } + + + + + + { + validator: { + $jsonSchema: { + bsonType: "object", + required: ["dossierId", "fileId", "analysisVersion", "analysisNumber", "entityLogEntryDocument", "legalBasis"], + properties: { + dossierId: { + bsonType: "string", + description: "The Dossier ID" + }, + fileId: { + bsonType: "string", + description: "The File ID" + }, + analysisVersion: { + bsonType: "long", + description: "The Analysis Version" + }, + analysisNumber: { + bsonType: "int", + description: "The Analysis Number" + }, + entityLogEntryDocument: { + bsonType: "array", + description: "The Entity Log Entry Documents", + items: { + bsonType: "objectId" + } + }, + legalBasis: { + bsonType: "array", + description: "The Legal Basis", + items: { + bsonType: "object" + } + }, + dictionaryVersion: { + bsonType: "long", + description: "The Dictionary Version" + }, + dossierDictionaryVersion: { + bsonType: "long", + description: "The Dossier Dictionary Version" + }, + rulesVersion: { + bsonType: "long", + description: "The Rules Version" + }, + legalBasisVersion: { + bsonType: "long", + description: "The Legal Basis Version" + } + } + } + }, + validationAction: "warn", + validationLevel: "strict" + } + + + + + + + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog.xml new file mode 100644 index 00000000..c8e1cbce --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/2-create-indices-for-entries.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/2-create-indices-for-entries.xml new file mode 100644 index 00000000..6667d00c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/2-create-indices-for-entries.xml @@ -0,0 +1,48 @@ + + + + + + + { + "entityLogId": 1, + } + + + {name: "entityLogId_index"} + + + + + + { + "entityLogId": 1, + "positions.pageNumber": 1 + } + + + {name: "entityLogId_positionsPageNumber_index"} + + + + + + { + "entityLogId": 1, + "containingNodeId": 1 + } + + + {name: "entityLogId_containingNodeId_index"} + + + + + + + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-create-indices-for-entries.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-create-indices-for-entries.xml new file mode 100644 index 00000000..d32c5010 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-create-indices-for-entries.xml @@ -0,0 +1,84 @@ + + + + + + + { + "entityLogId": 1, + } + + + {name: "entityLogId_index"} + + + + + + { + "entityLogId": 1, + "positions.pageNumber": 1 + } + + + {name: "entityLogId_positionsPageNumber_index"} + + + + + + { + "entityLogId": 1, + "changes.analysisNumber": -1 + } + + + {name: "entityLogId_changesAnalysisNumber_index"} + + + + + + { + "entityLogId": 1, + "containingNodeId": 1 + } + + + {name: "entityLogId_containingNodeId_index"} + + + + + + { + "id": 1, + "containingNodeId": 1 + } + + + {name: "id_containingNodeId_index"} + + + + + + { + "entityLogId": 1, + "type": 1 + } + + + {name: "entityLogId_type_index"} + + + + + + + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-remove-entry-number-of-comments.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-remove-entry-number-of-comments.xml new file mode 100644 index 00000000..9665c17b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-remove-entry-number-of-comments.xml @@ -0,0 +1,26 @@ + + + + + + { + update: "entity-log-entries", + updates: [ + { + q: {}, + u: { $unset: { "numberOfComments": "" } }, + multi: true + } + ] + } + + + + + + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index e606d38c..e116014b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -1,5 +1,9 @@ package com.iqser.red.service.redaction.v1.server; +import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_DATABASE; +import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_PASSWORD; +import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_USERNAME; +import static com.knecon.fforesight.tenantcommons.model.TenantResponse.builder; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -55,12 +59,13 @@ import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; +import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor; import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; import com.knecon.fforesight.tenantcommons.TenantContext; -import com.knecon.fforesight.tenantcommons.TenantsClient; +import com.knecon.fforesight.tenantcommons.TenantProvider; import com.knecon.fforesight.tenantcommons.model.MongoDBConnection; import com.mongodb.MongoCommandException; import com.mongodb.client.MongoClient; @@ -152,10 +157,16 @@ public abstract class AbstractRedactionIntegrationTest { protected LegalBasisClient legalBasisClient; @MockBean - private TenantsClient tenantsClient; + private MongoConnectionProvider mongoConnectionProvider; @MockBean - private MongoConnectionProvider mongoConnectionProvider; + private TenantProvider tenantProvider; + + @Autowired + protected MongoTestConfig mongoTestConfig; + + @Autowired + protected TenantMongoLiquibaseExecutor tenantMongoLiquibaseExecutor; protected final Map> dictionary = new HashMap<>(); protected final Map> dossierDictionary = new HashMap<>(); @@ -180,6 +191,28 @@ public abstract class AbstractRedactionIntegrationTest { protected DictionaryClient dictionaryClient; + @BeforeEach + public void setup() { + + TenantContext.setTenantId("redaction"); + + var mongoInstance = MongoDBTestContainer.getInstance(); + MongoDBConnection mongoDBConnection = MongoDBConnection.builder() + .prefix("mongodb") + .username(MONGO_USERNAME) + .password(MONGO_PASSWORD) + .address(mongoInstance.getHost() + ":" + mongoInstance.getFirstMappedPort()) + .database(MONGO_DATABASE) + .options("") + .build(); + + when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(mongoDBConnection); + when(tenantProvider.getTenant(any())).thenReturn(builder().tenantId("redaction").mongoDBConnection(mongoDBConnection).build()); + + tenantMongoLiquibaseExecutor.initializeTenant("redaction"); + } + + @AfterEach public void cleanupStorage() { @@ -583,23 +616,6 @@ public abstract class AbstractRedactionIntegrationTest { } - @BeforeEach - protected void mockProvideMongoDBConnection() { - - TenantContext.setTenantId("redaction"); - - var mongoInstance = MongoDBTestContainer.getInstance(); - - when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder() - .host(mongoInstance.getHost()) - .port(String.valueOf(mongoInstance.getFirstMappedPort())) - .database(MongoDBTestContainer.MONGO_DATABASE) - .username(MongoDBTestContainer.MONGO_USERNAME) - .password(MongoDBTestContainer.MONGO_PASSWORD) - .build()); - } - - @Slf4j static class Initializer implements ApplicationContextInitializer { @@ -613,8 +629,8 @@ public abstract class AbstractRedactionIntegrationTest { TestPropertyValues.of("MONGODB_HOST=" + mongoInstance.getHost(), "MONGODB_PORT=" + mongoInstance.getFirstMappedPort(), - "MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME, - "MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD).applyTo(configurableApplicationContext.getEnvironment()); + "MONGODB_USER=" + MONGO_USERNAME, + "MONGODB_PASSWORD=" + MONGO_PASSWORD).applyTo(configurableApplicationContext.getEnvironment()); } @@ -624,16 +640,16 @@ public abstract class AbstractRedactionIntegrationTest { private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) { try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/", - MongoDBTestContainer.MONGO_USERNAME, - MongoDBTestContainer.MONGO_PASSWORD, + MONGO_USERNAME, + MONGO_PASSWORD, mongoDBTestContainer.getHost(), mongoDBTestContainer.getFirstMappedPort()))) { - MongoDatabase database = mongoClient.getDatabase(MongoDBTestContainer.MONGO_DATABASE); + MongoDatabase database = mongoClient.getDatabase(MONGO_DATABASE); BsonDocument createUserCommand = new BsonDocument(); - createUserCommand.append("createUser", new BsonString(MongoDBTestContainer.MONGO_USERNAME)); - createUserCommand.append("pwd", new BsonString(MongoDBTestContainer.MONGO_PASSWORD)); + createUserCommand.append("createUser", new BsonString(MONGO_USERNAME)); + createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD)); BsonArray roles = new BsonArray(); - roles.add(new BsonString("readWrite")); + roles.add(new BsonString("dbOwner")); createUserCommand.append("roles", roles); try { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MongoTestConfig.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MongoTestConfig.java new file mode 100644 index 00000000..0e5c515d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MongoTestConfig.java @@ -0,0 +1,11 @@ +package com.iqser.red.service.redaction.v1.server; + +import org.springframework.context.annotation.Configuration; + +import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase; + +@Configuration +@EnableMongoLiquibase +public class MongoTestConfig { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java index 701c4eea..66a2e1f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java @@ -7,7 +7,7 @@ public final class MongoDBTestContainer extends GenericContainer