diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 4633dbbf..a4790e05 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.611.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index b0475f82..3c7b9b21 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,8 +16,8 @@ val layoutParserVersion = "0.181.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.611.0" -val llmServiceVersion = "1.11.0" +val persistenceServiceVersion = "2.612.0-RED10072.1" +val llmServiceVersion = "1.20.0-RED10072.2" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java index 5d1f8b17..fd08f339 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java @@ -82,4 +82,14 @@ public class NerEntities { LLM_NER } + + public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine mapToPrimaryEngine(NerEntities.Engine nerEntityEngine) { + + return switch (nerEntityEngine) { + case NER, CLOUD_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER; + case LLM_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.LLM_NER; + }; + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java index fa736613..9b79efde 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java @@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; @@ -39,6 +40,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.knecon.fforesight.llm.service.LlmNerEntities; import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; @@ -295,8 +297,10 @@ public class AnalysisPreparationService { private NerEntities getLlmNerEntities(AnalyzeRequest analyzeRequest) { - return new NerEntities(redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntities() + LlmNerEntities llmNerEntities = redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + return new NerEntities(llmNerEntities.getEntities() .stream() + //.filter(e -> allTypes.contains(e.getType())) .map(e -> new NerEntities.NerEntity(e.getValue(), new TextRange(e.getStartOffset(), e.getEndOffset()), e.getType(), diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 6f2eaf06..5e6485ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -20,6 +20,7 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java index c437d4d0..c5a8507f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.service; import java.awt.Color; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedList; @@ -497,4 +498,12 @@ public class DictionaryService { .collect(Collectors.toList()); } + + public List getAllTypes(String dossierTemplateId, String dossierId) { + + List allTypes = dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, null, false); + allTypes.addAll(dictionaryClient.getAllTypesForDossier(dossierId, null, false)); + return allTypes; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index a6c592b5..156ffb54 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -9,6 +9,7 @@ import static com.iqser.red.service.redaction.v1.server.service.document.EntityC import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween; import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -1176,7 +1177,7 @@ public class EntityCreationService { */ public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException( + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException( "No entity present!")); } @@ -1192,7 +1193,11 @@ public class EntityCreationService { */ public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!")); + return byTextRangeWithEngine(nerEntity.textRange(), + type, + entityType, + semanticNode, + getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException("No entity present!")); } @@ -1206,7 +1211,7 @@ public class EntityCreationService { */ public Optional optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1221,7 +1226,7 @@ public class EntityCreationService { */ public Optional optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { - return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1240,7 +1245,7 @@ public class EntityCreationService { if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) { return Optional.empty(); } - return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1264,7 +1269,7 @@ public class EntityCreationService { if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) { return Optional.empty(); } - return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)); + return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity)); } @@ -1280,7 +1285,7 @@ public class EntityCreationService { public Stream combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) { return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1307,7 +1312,7 @@ public class EntityCreationService { int minPartsToCombine) { return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, minPartsToCombine) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1336,7 +1341,7 @@ public class EntityCreationService { int minPartsToCombine) { return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1372,7 +1377,7 @@ public class EntityCreationService { maxDistanceBetweenParts, minPartsToCombine, minEssentialTypesCombined) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1411,7 +1416,7 @@ public class EntityCreationService { minPartsToCombine, minEssentialTypesCombined, confidence) - .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER))) + .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities))) .filter(Optional::isPresent) .map(Optional::get); } @@ -1440,7 +1445,7 @@ public class EntityCreationService { DocumentTree documentTree = node.getDocumentTree(); if (node.getEntities().contains(entity)) { - // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges + // If entity already exists, and it has a different text range, we add the text range to the list of duplicated text ranges node.getEntities() .stream()// .filter(e -> e.equals(entity))// @@ -1455,6 +1460,23 @@ public class EntityCreationService { } + private static Set getNerEngines(NerEntities nerEntities) { + + return getNerEngines(nerEntities.getNerEntityList() + .toArray(new NerEntities.NerEntity[0])); + } + + + private static Set getNerEngines(NerEntities.NerEntity... nerEntities) { + + return Arrays.stream(nerEntities) + .map(NerEntities.NerEntity::engine) + .map(NerEntities::mapToPrimaryEngine) + .collect(Collectors.toSet()); + } + + + private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) { entityToDuplicate.addTextRange(newTextRange); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 7c17a0ee..a352806a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -6,6 +6,7 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi import java.io.File; import java.io.FileInputStream; import java.io.InputStream; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; @@ -31,7 +32,6 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.document.DocumentData; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; -import com.iqser.red.storage.commons.exception.StorageException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.llm.service.LlmNerEntities; @@ -137,13 +137,13 @@ public class RedactionStorageService { } - @SneakyThrows public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) { componentLogMongoService.saveComponentLog(dossierId, fileId, componentLog); } + @SneakyThrows public void updateEntityLogEntries(String dossierId, String fileId, List entityLogEntries) { @@ -373,7 +373,12 @@ public class RedactionStorageService { public LlmNerEntities getLlmNerEntities(String dossierId, String fileId) { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES), LlmNerEntities.class); + String objectId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES); + if (storageService.objectExists(TenantContext.getTenantId(), objectId)) { + return storageService.readJSONObject(TenantContext.getTenantId(), objectId, LlmNerEntities.class); + } else { + return LlmNerEntities.builder().entities(new ArrayList<>()).build(); + } }