Merge branch 'feature/RED-10072' into 'master'

RED-10072: AI description field and toggle for entities

Closes RED-10072

See merge request redactmanager/redaction-service!539

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.611.0"
+val persistenceServiceVersion = "2.612.0-RED10072.1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -16,8 +16,8 @@ val layoutParserVersion = "0.181.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.611.0"
+val persistenceServiceVersion = "2.612.0-RED10072.1"
-val llmServiceVersion = "1.11.0"
+val llmServiceVersion = "1.20.0-RED10072.2"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java

@@ -82,4 +82,14 @@ public class NerEntities {
         LLM_NER
     }
 
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine mapToPrimaryEngine(NerEntities.Engine nerEntityEngine) {
+
+        return switch (nerEntityEngine) {
+            case NER, CLOUD_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case LLM_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.LLM_NER;
+        };
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java

@@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
@@ -39,6 +40,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS
 import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.knecon.fforesight.llm.service.LlmNerEntities;
 
 import io.micrometer.core.annotation.Timed;
 import io.micrometer.observation.annotation.Observed;
@@ -295,8 +297,10 @@ public class AnalysisPreparationService {
 
     private NerEntities getLlmNerEntities(AnalyzeRequest analyzeRequest) {
 
-        return new NerEntities(redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntities()
+        LlmNerEntities llmNerEntities = redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        return new NerEntities(llmNerEntities.getEntities()
                                        .stream()
+                                       //.filter(e -> allTypes.contains(e.getType()))
                                        .map(e -> new NerEntities.NerEntity(e.getValue(),
                                                                            new TextRange(e.getStartOffset(), e.getEndOffset()),
                                                                            e.getType(),

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -20,6 +20,7 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.awt.Color;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -497,4 +498,12 @@ public class DictionaryService {
                 .collect(Collectors.toList());
     }
 
+
+    public List<Type> getAllTypes(String dossierTemplateId, String dossierId) {
+
+        List<Type> allTypes = dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, null, false);
+        allTypes.addAll(dictionaryClient.getAllTypesForDossier(dossierId, null, false));
+        return allTypes;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -9,6 +9,7 @@ import static com.iqser.red.service.redaction.v1.server.service.document.EntityC
 import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween;
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -1176,7 +1177,7 @@ public class EntityCreationService {
      */
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException(
                 "No entity present!"));
     }
 
@@ -1192,7 +1193,11 @@ public class EntityCreationService {
      */
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
+        return byTextRangeWithEngine(nerEntity.textRange(),
+                                     type,
+                                     entityType,
+                                     semanticNode,
+                                     getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException("No entity present!"));
     }
 
 
@@ -1206,7 +1211,7 @@ public class EntityCreationService {
      */
     public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1221,7 +1226,7 @@ public class EntityCreationService {
      */
     public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1240,7 +1245,7 @@ public class EntityCreationService {
         if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
             return Optional.empty();
         }
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1264,7 +1269,7 @@ public class EntityCreationService {
         if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
             return Optional.empty();
         }
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
     }
 
 
@@ -1280,7 +1285,7 @@ public class EntityCreationService {
     public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
 
         return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1307,7 +1312,7 @@ public class EntityCreationService {
                                                  int minPartsToCombine) {
 
         return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, minPartsToCombine)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1336,7 +1341,7 @@ public class EntityCreationService {
                                                  int minPartsToCombine) {
 
         return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1372,7 +1377,7 @@ public class EntityCreationService {
                                                                     maxDistanceBetweenParts,
                                                                     minPartsToCombine,
                                                                     minEssentialTypesCombined)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1411,7 +1416,7 @@ public class EntityCreationService {
                                                                                   minPartsToCombine,
                                                                                   minEssentialTypesCombined,
                                                                                   confidence)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -1440,7 +1445,7 @@ public class EntityCreationService {
 
         DocumentTree documentTree = node.getDocumentTree();
         if (node.getEntities().contains(entity)) {
-            // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
+            // If entity already exists, and it has a different text range, we add the text range to the list of duplicated text ranges
             node.getEntities()
                     .stream()//
                     .filter(e -> e.equals(entity))//
@@ -1455,6 +1460,23 @@ public class EntityCreationService {
     }
 
 
+    private static Set<Engine> getNerEngines(NerEntities nerEntities) {
+
+        return getNerEngines(nerEntities.getNerEntityList()
+                                     .toArray(new NerEntities.NerEntity[0]));
+    }
+
+
+    private static Set<Engine> getNerEngines(NerEntities.NerEntity... nerEntities) {
+
+        return Arrays.stream(nerEntities)
+                .map(NerEntities.NerEntity::engine)
+                .map(NerEntities::mapToPrimaryEngine)
+                .collect(Collectors.toSet());
+    }
+
+
+
     private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
 
         entityToDuplicate.addTextRange(newTextRange);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -6,6 +6,7 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;
@@ -31,7 +32,6 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
-import com.iqser.red.storage.commons.exception.StorageException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.llm.service.LlmNerEntities;
@@ -137,13 +137,13 @@ public class RedactionStorageService {
     }
 
 
-
     @SneakyThrows
     public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) {
 
         componentLogMongoService.saveComponentLog(dossierId, fileId, componentLog);
     }
 
+
     @SneakyThrows
     public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
 
@@ -373,7 +373,12 @@ public class RedactionStorageService {
 
     public LlmNerEntities getLlmNerEntities(String dossierId, String fileId) {
 
-        return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES), LlmNerEntities.class);
+        String objectId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES);
+        if (storageService.objectExists(TenantContext.getTenantId(), objectId)) {
+            return storageService.readJSONObject(TenantContext.getTenantId(), objectId, LlmNerEntities.class);
+        } else {
+            return LlmNerEntities.builder().entities(new ArrayList<>()).build();
+        }
     }