Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8702: Explore document databases to store entityLog

Browse Source
This commit is contained in:
Maverick Studer 2024-04-11 16:42:36 +02:00
parent e9043c930a
commit 033fe17482
15 changed files with 487 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
dependencies {
implementation("org.springframework:spring-web:6.0.12")
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.383.0")
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.393.0")
}
publishing {

4
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.107.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.383.0"
val persistenceServiceVersion = "2.393.0"
val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4"
val testContainersVersion = "1.19.7"
@ -41,7 +41,7 @@ dependencies {
implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
implementation("com.iqser.red.commons:storage-commons:2.45.0")
implementation("com.knecon.fforesight:tenant-commons:0.23.0")
implementation("com.knecon.fforesight:tenant-commons:0.24.0")
implementation("com.knecon.fforesight:tracing-commons:0.5.0")
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java
View File

@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongo
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
import io.micrometer.core.aop.TimedAspect;
@ -34,6 +35,7 @@ import io.micrometer.observation.aop.ObservedAspect;
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
@EnableMongoLiquibase
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
public class Application {

6
redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml
View File

@ -20,3 +20,9 @@ redaction-service:
application:
type: "RedactManager"
multitenancy:
tenant:
mongo:
liquibase:
changeLog: classpath:mongo/changelog/mongo.changelog-tenant.xml

7
redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml
View File

@ -75,3 +75,10 @@ storage:
application:
type: "RedactManager"
multitenancy:
tenant:
mongo:
liquibase:
changeLog: classpath:mongo/changelog/mongo.changelog-tenant.xml

7
redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/mongo.changelog-tenant.xml Normal file
View File

@ -0,0 +1,7 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd">
<include file="/mongo/changelog/tenant/1-initial-database.changelog.xml"/>
<include file="/mongo/changelog/tenant/2-create-indices-for-entries.xml"/>
</databaseChangeLog>

224
redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog-with-validation.xml Normal file
View File

@ -0,0 +1,224 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="createEntityLogEntryCollection" author="maverick">
<ext:createCollection collectionName="entity-log-entries">
<ext:options>
{
validator: {
$jsonSchema: {
bsonType: "object",
required: ["entryId", "entityLogId", "type", "entryType", "state", "value", "reason", "matchedRule", "legalBasis", "containingNodeId", "closestHeadline", "section",
"positions", "textBefore", "textAfter", "startOffset", "endOffset", "imageHasTransparency", "dictionaryEntry", "dossierDictionaryEntry", "excluded", "changes",
"manualChanges", "engines", "reference", "importedRedactionIntersections", "numberOfComments"],
properties: {
entryId: {
bsonType: "string",
description: "The Entry ID"
},
entityLogId: {
bsonType: "string",
description: "The Entity Log ID"
},
type: {
bsonType: "string",
description: "The Type"
},
entryType: {
bsonType: "string",
description: "The Entry Type"
},
state: {
bsonType: "string",
description: "The Entry State"
},
value: {
bsonType: "string",
description: "The Value"
},
reason: {
bsonType: "string",
description: "The Reason"
},
matchedRule: {
bsonType: "string",
description: "The Matched Rule"
},
legalBasis: {
bsonType: "string",
description: "The Legal Basis"
},
containingNodeId: {
bsonType: "array",
items: {
bsonType: "int",
description: "The Containing Node ID"
}
},
closestHeadline: {
bsonType: "string",
description: "The Closest Headline"
},
section: {
bsonType: "string",
description: "The Section"
},
positions: {
bsonType: "array",
description: "The Positions",
items: {
bsonType: "object"
}
},
textBefore: {
bsonType: "string",
description: "Text before the entry"
},
textAfter: {
bsonType: "string",
description: "Text after the entry"
},
startOffset: {
bsonType: "int",
description: "Start offset of the entry"
},
endOffset: {
bsonType: "int",
description: "End offset of the entry"
},
imageHasTransparency: {
bsonType: "bool",
description: "Whether the image has transparency"
},
dictionaryEntry: {
bsonType: "bool",
description: "Whether it's a dictionary entry"
},
dossierDictionaryEntry: {
bsonType: "bool",
description: "Whether it's a dossier dictionary entry"
},
excluded: {
bsonType: "bool",
description: "Whether it's excluded"
},
changes: {
bsonType: "array",
description: "The Changes",
items: {
bsonType: "object"
}
},
manualChanges: {
bsonType: "array",
description: "The Manual Changes",
items: {
bsonType: "object"
}
},
engines: {
bsonType: "array",
description: "The Engines",
items: {
bsonType: "string"
}
},
reference: {
bsonType: "array",
description: "The Reference",
items: {
bsonType: "string"
}
},
importedRedactionIntersections: {
bsonType: "array",
description: "The Imported Redaction Intersections",
items: {
bsonType: "string"
}
},
numberOfComments: {
bsonType: "int",
description: "The Number of Comments"
}
}
}
},
validationAction: "warn",
validationLevel: "strict"
}
</ext:options>
</ext:createCollection>
<ext:createCollection collectionName="entity-logs">
<ext:options>
{
validator: {
$jsonSchema: {
bsonType: "object",
required: ["dossierId", "fileId", "analysisVersion", "analysisNumber", "entityLogEntryDocument", "legalBasis"],
properties: {
dossierId: {
bsonType: "string",
description: "The Dossier ID"
},
fileId: {
bsonType: "string",
description: "The File ID"
},
analysisVersion: {
bsonType: "long",
description: "The Analysis Version"
},
analysisNumber: {
bsonType: "int",
description: "The Analysis Number"
},
entityLogEntryDocument: {
bsonType: "array",
description: "The Entity Log Entry Documents",
items: {
bsonType: "objectId"
}
},
legalBasis: {
bsonType: "array",
description: "The Legal Basis",
items: {
bsonType: "object"
}
},
dictionaryVersion: {
bsonType: "long",
description: "The Dictionary Version"
},
dossierDictionaryVersion: {
bsonType: "long",
description: "The Dossier Dictionary Version"
},
rulesVersion: {
bsonType: "long",
description: "The Rules Version"
},
legalBasisVersion: {
bsonType: "long",
description: "The Legal Basis Version"
}
}
}
},
validationAction: "warn",
validationLevel: "strict"
}
</ext:options>
</ext:createCollection>
</changeSet>
</databaseChangeLog>

17
redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/1-initial-database.changelog.xml Normal file
View File

@ -0,0 +1,17 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="createEntityLogEntryCollection" author="maverick">
<ext:createCollection collectionName="entity-log-entries"/>
<ext:createCollection collectionName="entity-logs"/>
</changeSet>
</databaseChangeLog>

48
redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/2-create-indices-for-entries.xml Normal file
View File

@ -0,0 +1,48 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="createIndicesForEntries" author="maverick">
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
}
</ext:keys>
<ext:options>
{name: "entityLogId_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
"positions.pageNumber": 1
}
</ext:keys>
<ext:options>
{name: "entityLogId_positionsPageNumber_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
"containingNodeId": 1
}
</ext:keys>
<ext:options>
{name: "entityLogId_containingNodeId_index"}
</ext:options>
</ext:createIndex>
</changeSet>
</databaseChangeLog>

84
redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-create-indices-for-entries.xml Normal file
View File

@ -0,0 +1,84 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="createIndicesForEntries" author="maverick">
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
}
</ext:keys>
<ext:options>
{name: "entityLogId_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
"positions.pageNumber": 1
}
</ext:keys>
<ext:options>
{name: "entityLogId_positionsPageNumber_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
"changes.analysisNumber": -1
}
</ext:keys>
<ext:options>
{name: "entityLogId_changesAnalysisNumber_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
"containingNodeId": 1
}
</ext:keys>
<ext:options>
{name: "entityLogId_containingNodeId_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"id": 1,
"containingNodeId": 1
}
</ext:keys>
<ext:options>
{name: "id_containingNodeId_index"}
</ext:options>
</ext:createIndex>
<ext:createIndex collectionName="entity-log-entries">
<ext:keys>
{
"entityLogId": 1,
"type": 1
}
</ext:keys>
<ext:options>
{name: "entityLogId_type_index"}
</ext:options>
</ext:createIndex>
</changeSet>
</databaseChangeLog>

26
redaction-service-v1/redaction-service-server-v1/src/main/resources/mongo/changelog/tenant/example-remove-entry-number-of-comments.xml Normal file
View File

@ -0,0 +1,26 @@
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
<changeSet id="1" author="maverick">
<ext:runCommand>
<ext:command>
{
update: "entity-log-entries",
updates: [
{
q: {},
u: { $unset: { "numberOfComments": "" } },
multi: true
}
]
}
</ext:command>
</ext:runCommand>
</changeSet>
</databaseChangeLog>

72
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -1,5 +1,9 @@
package com.iqser.red.service.redaction.v1.server;
import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_DATABASE;
import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_PASSWORD;
import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_USERNAME;
import static com.knecon.fforesight.tenantcommons.model.TenantResponse.builder;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@ -55,12 +59,13 @@ import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor;
import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import com.knecon.fforesight.tenantcommons.TenantProvider;
import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
import com.mongodb.MongoCommandException;
import com.mongodb.client.MongoClient;
@ -152,10 +157,16 @@ public abstract class AbstractRedactionIntegrationTest {
protected LegalBasisClient legalBasisClient;
@MockBean
private TenantsClient tenantsClient;
private MongoConnectionProvider mongoConnectionProvider;
@MockBean
private MongoConnectionProvider mongoConnectionProvider;
private TenantProvider tenantProvider;
@Autowired
protected MongoTestConfig mongoTestConfig;
@Autowired
protected TenantMongoLiquibaseExecutor tenantMongoLiquibaseExecutor;
protected final Map<String, List<String>> dictionary = new HashMap<>();
protected final Map<String, List<String>> dossierDictionary = new HashMap<>();
@ -180,6 +191,28 @@ public abstract class AbstractRedactionIntegrationTest {
protected DictionaryClient dictionaryClient;
@BeforeEach
public void setup() {
TenantContext.setTenantId("redaction");
var mongoInstance = MongoDBTestContainer.getInstance();
MongoDBConnection mongoDBConnection = MongoDBConnection.builder()
.prefix("mongodb")
.username(MONGO_USERNAME)
.password(MONGO_PASSWORD)
.address(mongoInstance.getHost() + ":" + mongoInstance.getFirstMappedPort())
.database(MONGO_DATABASE)
.options("")
.build();
when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(mongoDBConnection);
when(tenantProvider.getTenant(any())).thenReturn(builder().tenantId("redaction").mongoDBConnection(mongoDBConnection).build());
tenantMongoLiquibaseExecutor.initializeTenant("redaction");
}
@AfterEach
public void cleanupStorage() {
@ -583,23 +616,6 @@ public abstract class AbstractRedactionIntegrationTest {
}
@BeforeEach
protected void mockProvideMongoDBConnection() {
TenantContext.setTenantId("redaction");
var mongoInstance = MongoDBTestContainer.getInstance();
when(mongoConnectionProvider.getMongoDBConnection(any())).thenReturn(MongoDBConnection.builder()
.host(mongoInstance.getHost())
.port(String.valueOf(mongoInstance.getFirstMappedPort()))
.database(MongoDBTestContainer.MONGO_DATABASE)
.username(MongoDBTestContainer.MONGO_USERNAME)
.password(MongoDBTestContainer.MONGO_PASSWORD)
.build());
}
@Slf4j
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
@ -613,8 +629,8 @@ public abstract class AbstractRedactionIntegrationTest {
TestPropertyValues.of("MONGODB_HOST=" + mongoInstance.getHost(),
"MONGODB_PORT=" + mongoInstance.getFirstMappedPort(),
"MONGODB_USER=" + MongoDBTestContainer.MONGO_USERNAME,
"MONGODB_PASSWORD=" + MongoDBTestContainer.MONGO_PASSWORD).applyTo(configurableApplicationContext.getEnvironment());
"MONGODB_USER=" + MONGO_USERNAME,
"MONGODB_PASSWORD=" + MONGO_PASSWORD).applyTo(configurableApplicationContext.getEnvironment());
}
@ -624,16 +640,16 @@ public abstract class AbstractRedactionIntegrationTest {
private static void createMongoDBDatabase(MongoDBTestContainer mongoDBTestContainer) {
try (MongoClient mongoClient = MongoClients.create(String.format("mongodb://%s:%s@%s:%s/",
MongoDBTestContainer.MONGO_USERNAME,
MongoDBTestContainer.MONGO_PASSWORD,
MONGO_USERNAME,
MONGO_PASSWORD,
mongoDBTestContainer.getHost(),
mongoDBTestContainer.getFirstMappedPort()))) {
MongoDatabase database = mongoClient.getDatabase(MongoDBTestContainer.MONGO_DATABASE);
MongoDatabase database = mongoClient.getDatabase(MONGO_DATABASE);
BsonDocument createUserCommand = new BsonDocument();
createUserCommand.append("createUser", new BsonString(MongoDBTestContainer.MONGO_USERNAME));
createUserCommand.append("pwd", new BsonString(MongoDBTestContainer.MONGO_PASSWORD));
createUserCommand.append("createUser", new BsonString(MONGO_USERNAME));
createUserCommand.append("pwd", new BsonString(MONGO_PASSWORD));
BsonArray roles = new BsonArray();
roles.add(new BsonString("readWrite"));
roles.add(new BsonString("dbOwner"));
createUserCommand.append("roles", roles);
try {

11
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MongoTestConfig.java Normal file
View File

@ -0,0 +1,11 @@
package com.iqser.red.service.redaction.v1.server;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
@Configuration
@EnableMongoLiquibase
public class MongoTestConfig {
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/testcontainers/MongoDBTestContainer.java
View File

@ -7,7 +7,7 @@ public final class MongoDBTestContainer extends GenericContainer<MongoDBTestCont
private static final String IMAGE_VERSION = "mongo:7.0.2";
public static final Integer MONGO_PORT = 27017;
public static final String MONGO_DATABASE = "mongo_database";
public static final String MONGO_DATABASE = "redaction";
public static final String MONGO_PASSWORD = "mongo_password";
public static final String MONGO_USERNAME = "mongo_username";
private static MongoDBTestContainer mongoDB;

7
redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml
View File

@ -44,3 +44,10 @@ management:
health.enabled: true
endpoints.web.exposure.include: prometheus, health, metrics
metrics.export.prometheus.enabled: true
multitenancy:
tenant:
mongo:
liquibase:
changeLog: classpath:mongo/changelog/mongo.changelog-tenant.xml