RED-8702: Explore document databases to store entityLog

* bugfix

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -42,7 +42,7 @@ dependencies {
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
     implementation("com.knecon.fforesight:tenant-commons:maverick-mongo")
-    implementation("com.knecon.fforesight:mongo-database-commons:maverick-mongo3")
+    implementation("com.knecon.fforesight:mongo-database-commons:maverick-mongo6")
     implementation("com.knecon.fforesight:tracing-commons:0.5.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
@@ -67,9 +67,9 @@ dependencies {
 
     implementation("org.reflections:reflections:0.10.2")
 
-    implementation("org.liquibase:liquibase-core:4.24.0")
+    //todo 8702: remove this
-
+    implementation("org.liquibase:liquibase-core:4.20.0")
-    implementation("org.liquibase.ext:liquibase-mongodb:4.24.0")
+    implementation("org.liquibase.ext:liquibase-mongodb:4.20.0")
 
     testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -4,8 +4,10 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
+import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
@@ -31,7 +33,7 @@ import io.micrometer.observation.aop.ObservedAspect;
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @EnableMongoRepositories
-@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class})
+@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
 public class Application {
 
     public static void main(String[] args) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/mongo/EntityLogMongoService.java

@@ -13,6 +13,8 @@ import com.iqser.red.service.persistence.management.v1.processor.document.Entity
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 
+import jakarta.transaction.Transactional;
+
 @Service
 public class EntityLogMongoService {
 
@@ -27,33 +29,40 @@ public class EntityLogMongoService {
     }
 
 
+    @Transactional
     public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
 
         EntityLogDocument entityLogDocument = entityLogDocumentRepository.insert(new EntityLogDocument(dossierId, fileId, entityLog));
 
         entityLogEntryDocumentRepository.insert(entityLog.getEntityLogEntry()
                                                         .stream()
-                                                        .map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry))
+                                                        .map(entityLogEntry -> new EntityLogEntryDocument(entityLogDocument.getId(), entityLogEntry)).distinct()
                                                         .toList());
     }
 
 
     // this does everything : insert when not found and update if found
     // todo: remove and replace when services use insert,update,delete correctly
+    @Transactional
     public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
 
         Optional<EntityLogDocument> optionalEntityLogDocument = entityLogDocumentRepository.findById(EntityLogDocument.getDocumentId(dossierId, fileId));
+        System.out.println("8702 DEBUG - save entity log");
         if (optionalEntityLogDocument.isEmpty()) {
+            System.out.println("8702 DEBUG - executing insert");
             // throw new EntityLogDocumentNotFoundException(String.format("Entity log for dossier %s and file %s not found.", dossierId, fileId));
             insertEntityLog(dossierId, fileId, entityLog);
             return;
         }
 
+        System.out.println("8702 DEBUG - executing update");
         EntityLogDocument oldEntityLogDocument = optionalEntityLogDocument.get();
         List<EntityLogEntryDocument> oldEntityLogEntryDocuments = oldEntityLogDocument.getEntityLogEntryDocument();
+        System.out.println("8702 DEBUG - oldEntityLogEntryDocuments " + oldEntityLogEntryDocuments.size());
 
         EntityLogDocument newEntityLogDocument = new EntityLogDocument(dossierId, fileId, entityLog);
         List<EntityLogEntryDocument> newEntityLogEntryDocuments = newEntityLogDocument.getEntityLogEntryDocument();
+        System.out.println("8702 DEBUG - newEntityLogEntryDocuments " + newEntityLogEntryDocuments.size());
 
         List<EntityLogEntryDocument> toUpdate = new ArrayList<>(newEntityLogEntryDocuments);
         toUpdate.retainAll(oldEntityLogEntryDocuments);
@@ -64,24 +73,31 @@ public class EntityLogMongoService {
         List<EntityLogEntryDocument> toInsert = new ArrayList<>(newEntityLogEntryDocuments);
         toInsert.removeAll(toUpdate);
 
-        entityLogEntryDocumentRepository.saveAll(toUpdate);
+        toInsert.forEach(System.out::println);
-        entityLogEntryDocumentRepository.deleteAll(toRemove);
-        entityLogEntryDocumentRepository.insert(toInsert);
 
+        System.out.println("8702 DEBUG - saving toUpdate " + toUpdate.size());
+        entityLogEntryDocumentRepository.saveAll(toUpdate);
+        System.out.println("8702 DEBUG - deleting toRemove " + toRemove.size());
+        entityLogEntryDocumentRepository.deleteAll(toRemove);
+        System.out.println("8702 DEBUG - inserting toInsert " + toInsert.size());
+        System.out.println("8702 DEBUG - inserting toInsert " + toInsert.stream().distinct().toList());
+        entityLogEntryDocumentRepository.insert(toInsert.stream().distinct().toList());
+
+        System.out.println("8702 DEBUG - saving newEntityLogDocument " + newEntityLogDocument.getEntityLogEntryDocument().size());
         entityLogDocumentRepository.save(newEntityLogDocument);
     }
 
 
+    @Transactional
     public void deleteEntityLog(String dossierId, String fileId) {
 
         String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
 
         entityLogDocumentRepository.deleteById(entityLogId);
-
         entityLogEntryDocumentRepository.deleteByEntityLogId(entityLogId);
     }
 
-
+    @Transactional
     public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
 
         String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);
@@ -109,6 +125,7 @@ public class EntityLogMongoService {
     }
 
 
+    @Transactional
     public void deleteEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
 
         String entityLogId = EntityLogDocument.getDocumentId(dossierId, fileId);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MyMongoLiquibaseTest.java

@@ -1,11 +1,19 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import static com.mongodb.client.model.Filters.eq;
+import java.io.File;
 
-import org.junit.jupiter.api.Disabled;
+import org.bson.Document;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.io.ResourceLoader;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
 
-import liquibase.Contexts;
 import liquibase.Liquibase;
 import liquibase.database.DatabaseFactory;
 import liquibase.ext.mongodb.database.MongoLiquibaseDatabase;
@@ -18,18 +26,22 @@ public class MyMongoLiquibaseTest {
     private static final String PORT = "27017";
     private static final String DATABASE = "redaction";
     private static final String USERNAME = "root";
-    private static final String PASSWORD = "AP8ckozpAl55JDAAFR4rxpbPj1WXdBTL";
+    private static final String PASSWORD = "XaRIBExTr8JrCSfgaHek3dl7wJj7dKZm";
 
     private static final String URL = String.format("mongodb://%s:%s/%s", HOSTNAME, PORT, DATABASE);
 
     @Test
-    @Disabled
+    //@Disabled
     @SneakyThrows
     public void testMongoDB() {
 
         try (MongoLiquibaseDatabase database = (MongoLiquibaseDatabase) DatabaseFactory.getInstance().openDatabase(URL, USERNAME, PASSWORD, null, null)) {
-            try (Liquibase liquibase = new Liquibase("mongo/liquibase/test.changelog.xml", new ClassLoaderResourceAccessor(), database)) {
+
-                liquibase.update();
+            //MongoCollection<Document> logs = database.getMongoDatabase().getCollection("entity-logs");
+            //long count = logs.countDocuments();
+            String path = "/mongo/liquibase/changelog/mongo.changelog-tenant.xml";
+            try (Liquibase liquibase = new Liquibase(path, new ClassLoaderResourceAccessor(), database)) {
+                liquibase.update("");
             }
         }
     }

redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/mongo.changelog-tenant.xml

@@ -0,0 +1,7 @@
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd">
+    <include file="/mongo/liquibase/changelog/tenant/1-initial-database.changelog.xml"/>
+
+</databaseChangeLog>

redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/1-initial-database.changelog-with-validation.xml

@@ -0,0 +1,224 @@
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <changeSet id="createEntityLogEntryCollection" author="maverick">
+
+        <ext:createCollection collectionName="entity-log-entries">
+            <ext:options>
+                {
+                    validator: {
+                        $jsonSchema: {
+                            bsonType: "object",
+                            required: ["entryId", "entityLogId", "type", "entryType", "state", "value", "reason", "matchedRule", "legalBasis", "containingNodeId", "closestHeadline", "section",
+                                "positions", "textBefore", "textAfter", "startOffset", "endOffset", "imageHasTransparency", "dictionaryEntry", "dossierDictionaryEntry", "excluded", "changes",
+                                "manualChanges", "engines", "reference", "importedRedactionIntersections", "numberOfComments"],
+                            properties: {
+                                entryId: {
+                                    bsonType: "string",
+                                    description: "The Entry ID"
+                                },
+                                    entityLogId: {
+                                    bsonType: "string",
+                                    description: "The Entity Log ID"
+                                },
+                                type: {
+                                    bsonType: "string",
+                                    description: "The Type"
+                                },
+                                entryType: {
+                                    bsonType: "string",
+                                    description: "The Entry Type"
+                                },
+                                state: {
+                                    bsonType: "string",
+                                    description: "The Entry State"
+                                },
+                                value: {
+                                    bsonType: "string",
+                                    description: "The Value"
+                                },
+                                reason: {
+                                    bsonType: "string",
+                                    description: "The Reason"
+                                },
+                                matchedRule: {
+                                    bsonType: "string",
+                                    description: "The Matched Rule"
+                                },
+                                legalBasis: {
+                                    bsonType: "string",
+                                    description: "The Legal Basis"
+                                },
+                                containingNodeId: {
+                                    bsonType: "array",
+                                    items: {
+                                        bsonType: "int",
+                                        description: "The Containing Node ID"
+                                    }
+                                },
+                                closestHeadline: {
+                                    bsonType: "string",
+                                    description: "The Closest Headline"
+                                },
+                                section: {
+                                    bsonType: "string",
+                                    description: "The Section"
+                                },
+                                positions: {
+                                    bsonType: "array",
+                                    description: "The Positions",
+                                    items: {
+                                        bsonType: "object"
+                                    }
+                                },
+                                textBefore: {
+                                    bsonType: "string",
+                                    description: "Text before the entry"
+                                },
+                                textAfter: {
+                                    bsonType: "string",
+                                    description: "Text after the entry"
+                                },
+                                startOffset: {
+                                    bsonType: "int",
+                                    description: "Start offset of the entry"
+                                },
+                                endOffset: {
+                                    bsonType: "int",
+                                    description: "End offset of the entry"
+                                },
+                                imageHasTransparency: {
+                                    bsonType: "bool",
+                                    description: "Whether the image has transparency"
+                                },
+                                dictionaryEntry: {
+                                    bsonType: "bool",
+                                    description: "Whether it's a dictionary entry"
+                                },
+                                dossierDictionaryEntry: {
+                                    bsonType: "bool",
+                                    description: "Whether it's a dossier dictionary entry"
+                                },
+                                excluded: {
+                                    bsonType: "bool",
+                                    description: "Whether it's excluded"
+                                },
+                                changes: {
+                                    bsonType: "array",
+                                    description: "The Changes",
+                                    items: {
+                                        bsonType: "object"
+                                    }
+                                },
+                                manualChanges: {
+                                    bsonType: "array",
+                                    description: "The Manual Changes",
+                                    items: {
+                                        bsonType: "object"
+                                    }
+                                },
+                                engines: {
+                                    bsonType: "array",
+                                    description: "The Engines",
+                                    items: {
+                                        bsonType: "string"
+                                    }
+                                },
+                                reference: {
+                                    bsonType: "array",
+                                    description: "The Reference",
+                                    items: {
+                                        bsonType: "string"
+                                    }
+                                },
+                                importedRedactionIntersections: {
+                                    bsonType: "array",
+                                    description: "The Imported Redaction Intersections",
+                                    items: {
+                                        bsonType: "string"
+                                    }
+                                },
+                                numberOfComments: {
+                                    bsonType: "int",
+                                    description: "The Number of Comments"
+                                }
+                            }
+                        }
+                    },
+                    validationAction: "warn",
+                    validationLevel: "strict"
+                }
+            </ext:options>
+        </ext:createCollection>
+
+        <ext:createCollection collectionName="entity-logs">
+            <ext:options>
+                {
+                    validator: {
+                        $jsonSchema: {
+                            bsonType: "object",
+                            required: ["dossierId", "fileId", "analysisVersion", "analysisNumber", "entityLogEntryDocument", "legalBasis"],
+                            properties: {
+                                dossierId: {
+                                    bsonType: "string",
+                                    description: "The Dossier ID"
+                                },
+                                fileId: {
+                                    bsonType: "string",
+                                    description: "The File ID"
+                                },
+                                analysisVersion: {
+                                    bsonType: "long",
+                                    description: "The Analysis Version"
+                                },
+                                analysisNumber: {
+                                    bsonType: "int",
+                                    description: "The Analysis Number"
+                                },
+                                entityLogEntryDocument: {
+                                    bsonType: "array",
+                                    description: "The Entity Log Entry Documents",
+                                    items: {
+                                        bsonType: "objectId"
+                                    }
+                                },
+                                legalBasis: {
+                                    bsonType: "array",
+                                    description: "The Legal Basis",
+                                    items: {
+                                        bsonType: "object"
+                                    }
+                                },
+                                dictionaryVersion: {
+                                    bsonType: "long",
+                                    description: "The Dictionary Version"
+                                },
+                                dossierDictionaryVersion: {
+                                    bsonType: "long",
+                                    description: "The Dossier Dictionary Version"
+                                },
+                                rulesVersion: {
+                                    bsonType: "long",
+                                    description: "The Rules Version"
+                                },
+                                legalBasisVersion: {
+                                    bsonType: "long",
+                                    description: "The Legal Basis Version"
+                                }
+                            }
+                        }
+                    },
+                    validationAction: "warn",
+                    validationLevel: "strict"
+                }
+            </ext:options>
+        </ext:createCollection>
+
+    </changeSet>
+
+
+</databaseChangeLog>

redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/1-initial-database.changelog.xml

@@ -0,0 +1,17 @@
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <changeSet id="createEntityLogEntryCollection" author="maverick">
+
+        <ext:createCollection collectionName="entity-log-entries"/>
+
+        <ext:createCollection collectionName="entity-logs"/>
+
+    </changeSet>
+
+
+</databaseChangeLog>

redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/2-create-indices-for-entries.xml

@@ -0,0 +1,37 @@
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <changeSet id="createEntityLogEntryCollection" author="maverick">
+
+        <ext:createIndex collectionName="entity-log-entries">
+            <ext:keys>
+                {
+                    "_id": 1,
+                    "positions.pageNumber": 1
+                }
+            </ext:keys>
+            <ext:options>
+                {name: "positions_pageNumber_index"}
+            </ext:options>
+        </ext:createIndex>
+
+        <ext:createIndex collectionName="entity-log-entries">
+            <ext:keys>
+                {
+                    "_id": 1,
+                    "changes.analysisNumber": -1
+                }
+            </ext:keys>
+            <ext:options>
+                {name: "changes_analysisNumber_index"}
+            </ext:options>
+        </ext:createIndex>
+
+    </changeSet>
+
+
+</databaseChangeLog>

redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/changelog/tenant/example-remove-entry-number-of-comments.changelog.xml

@@ -0,0 +1,26 @@
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.20.xsd
+        http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <changeSet id="1" author="maverick">
+        <ext:runCommand>
+            <ext:command>
+                {
+                    update: "entity-log-entries",
+                    updates: [
+                        {
+                            q: {},
+                            u: { $unset: { "numberOfComments": "" } },
+                            multi: true
+                        }
+                    ]
+                }
+            </ext:command>
+
+        </ext:runCommand>
+    </changeSet>
+
+</databaseChangeLog>

redaction-service-v1/redaction-service-server-v1/src/test/resources/mongo/liquibase/test.changelog.xml

@@ -1,45 +0,0 @@
-<!--
-  #%L
-  Liquibase MongoDB Extension
-  %%
-  Copyright (C) 2019 Mastercard
-  %%
-  Licensed under the Apache License, Version 2.0 (the "License").
-  You may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-  #L%
-  -->
-<databaseChangeLog
-        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xmlns:mongodb="http://www.liquibase.org/xml/ns/mongodb"
-        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.6.xsd
-        http://www.liquibase.org/xml/ns/mongodb http://www.liquibase.org/xml/ns/mongodb/liquibase-mongodb-latest.xsd">
-
-    <changeSet id="1" author="maverick">
-        <mongodb:runCommand>
-            <mongodb:command>
-                {
-                    update: "entity-logs",
-                    updates: [
-                        {
-                            q: {},
-                            u: { $rename: { "dId": "dossierId" } },
-                            multi: true
-                        }
-                    ]
-                }
-            </mongodb:command>
-
-        </mongodb:runCommand>
-    </changeSet>
-
-</databaseChangeLog>