Merge branch 'RED-8702-fix' into 'master'

RED-8702: Explore document databases to store entityLog

Closes RED-8702

See merge request redactmanager/redaction-service!371

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.393.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.395.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -16,7 +16,7 @@ val layoutParserVersion = "0.107.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.393.0"
+val persistenceServiceVersion = "2.395.0"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -23,7 +23,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
@@ -264,15 +263,16 @@ public class AnalyzeService {
 
         EntityLog entityLog = entityLogChanges.getEntityLog();
 
-        // analysis numbers should be incremented in every follow-up request, so that this could be replaced
+        // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
+        // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
         if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
-            redactionStorageService.insertEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+            redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
 
         } else {
             redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
 
             if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
-                redactionStorageService.insertEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
+                redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
             }
             if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
                 redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -81,13 +81,6 @@ public class RedactionStorageService {
     }
 
 
-    @SneakyThrows
-    public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
-
-        entityLogMongoService.upsertEntityLog(dossierId, fileId, entityLog);
-    }
-
-
     @SneakyThrows
     public void updateEntityLogWithoutEntries(String dossierId, String fileId, EntityLog entityLog) {
 
@@ -96,16 +89,16 @@ public class RedactionStorageService {
 
 
     @SneakyThrows
-    public void insertEntityLog(String dossierId, String fileId, EntityLog entityLog) {
+    public void saveEntityLog(String dossierId, String fileId, EntityLog entityLog) {
 
-        entityLogMongoService.insertEntityLog(dossierId, fileId, entityLog);
+        entityLogMongoService.saveEntityLog(dossierId, fileId, entityLog);
     }
 
 
     @SneakyThrows
-    public void insertEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
+    public void saveEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
 
-        entityLogMongoService.insertEntityLogEntries(dossierId, fileId, entityLogEntries);
+        entityLogMongoService.saveEntityLogEntries(dossierId, fileId, entityLogEntries);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -1514,6 +1514,45 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
     }
 
 
+    @Test
+    @Disabled
+    public void testPydiflumetofenMongo() throws IOException {
+
+        String fileName = "files/Pydiflumetofen/85 Pydiflumetofen_DAR_08_Volume_3CA_B-6_2017-07-26.pdf";
+
+        AnalyzeRequest request = uploadFileToStorage(fileName);
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+
+        analyzeService.analyze(request);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        dictionary.get(DICTIONARY_AUTHOR).add("EFSA");
+        reanlysisVersions.put("EFSA", 2L);
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
+        mockDictionaryCalls(0L);
+
+        analyzeService.reanalyze(request);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        dictionary.get(DICTIONARY_ADDRESS).add("SYN545974");
+        reanlysisVersions.put("SYN545974", 3L);
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
+        mockDictionaryCalls(2L);
+
+        analyzeService.reanalyze(request);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        dictionary.get(DOSSIER_REDACTIONS_INDICATOR).add("ToxRTool");
+        reanlysisVersions.put("ToxRTool", 4L);
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
+        mockDictionaryCalls(3L);
+
+        analyzeService.reanalyze(request);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+    }
+
+
     private IdRemoval getIdRemoval(String id) {
 
         return IdRemoval.builder()