Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'feature/RED-10072' into 'master'

Browse Source 
RED-10072: AI description field and toggle for entities

Closes RED-10072

See merge request redactmanager/redaction-service!539
This commit is contained in:
Maverick Studer 2024-11-07 14:43:51 +01:00
commit e415234bf8
8 changed files with 70 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -4,7 +4,7 @@ plugins {
}
description = "redaction-service-api-v1"
val persistenceServiceVersion = "2.611.0"
val persistenceServiceVersion = "2.612.0-RED10072.1"
dependencies {
implementation("org.springframework:spring-web:6.0.12")

4
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,8 +16,8 @@ val layoutParserVersion = "0.181.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.611.0"
val llmServiceVersion = "1.11.0"
val persistenceServiceVersion = "2.612.0-RED10072.1"
val llmServiceVersion = "1.20.0-RED10072.2"
val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4"
val testContainersVersion = "1.19.7"

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java
View File

@ -82,4 +82,14 @@ public class NerEntities {
LLM_NER
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine mapToPrimaryEngine(NerEntities.Engine nerEntityEngine) {
return switch (nerEntityEngine) {
case NER, CLOUD_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
case LLM_NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.LLM_NER;
};
}
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java
View File

@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
@ -39,6 +40,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderS
import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.knecon.fforesight.llm.service.LlmNerEntities;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
@ -295,8 +297,10 @@ public class AnalysisPreparationService {
private NerEntities getLlmNerEntities(AnalyzeRequest analyzeRequest) {
return new NerEntities(redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntities()
LlmNerEntities llmNerEntities = redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
return new NerEntities(llmNerEntities.getEntities()
.stream()
//.filter(e -> allTypes.contains(e.getType()))
.map(e -> new NerEntities.NerEntity(e.getValue(),
new TextRange(e.getStartOffset(), e.getEndOffset()),
e.getType(),

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -20,6 +20,7 @@ import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.awt.Color;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedList;
@ -497,4 +498,12 @@ public class DictionaryService {
.collect(Collectors.toList());
}
public List<Type> getAllTypes(String dossierTemplateId, String dossierId) {
List<Type> allTypes = dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, null, false);
allTypes.addAll(dictionaryClient.getAllTypesForDossier(dossierId, null, false));
return allTypes;
}
}

46
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java
View File

@ -9,6 +9,7 @@ import static com.iqser.red.service.redaction.v1.server.service.document.EntityC
import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.truncateEndIfLineBreakIsBetween;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@ -1176,7 +1177,7 @@ public class EntityCreationService {
*/
public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException(
"No entity present!"));
}
@ -1192,7 +1193,11 @@ public class EntityCreationService {
*/
public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
return byTextRangeWithEngine(nerEntity.textRange(),
type,
entityType,
semanticNode,
getNerEngines(nerEntity)).orElseThrow(() -> new NotFoundException("No entity present!"));
}
@ -1206,7 +1211,7 @@ public class EntityCreationService {
*/
public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1221,7 +1226,7 @@ public class EntityCreationService {
*/
public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1240,7 +1245,7 @@ public class EntityCreationService {
if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
return Optional.empty();
}
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1264,7 +1269,7 @@ public class EntityCreationService {
if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
return Optional.empty();
}
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, getNerEngines(nerEntity));
}
@ -1280,7 +1285,7 @@ public class EntityCreationService {
public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1307,7 +1312,7 @@ public class EntityCreationService {
int minPartsToCombine) {
return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, minPartsToCombine)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1336,7 +1341,7 @@ public class EntityCreationService {
int minPartsToCombine) {
return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1372,7 +1377,7 @@ public class EntityCreationService {
maxDistanceBetweenParts,
minPartsToCombine,
minEssentialTypesCombined)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1411,7 +1416,7 @@ public class EntityCreationService {
minPartsToCombine,
minEssentialTypesCombined,
confidence)
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
.map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, getNerEngines(nerEntities)))
.filter(Optional::isPresent)
.map(Optional::get);
}
@ -1440,7 +1445,7 @@ public class EntityCreationService {
DocumentTree documentTree = node.getDocumentTree();
if (node.getEntities().contains(entity)) {
// If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
// If entity already exists, and it has a different text range, we add the text range to the list of duplicated text ranges
node.getEntities()
.stream()//
.filter(e -> e.equals(entity))//
@ -1455,6 +1460,23 @@ public class EntityCreationService {
}
private static Set<Engine> getNerEngines(NerEntities nerEntities) {
return getNerEngines(nerEntities.getNerEntityList()
.toArray(new NerEntities.NerEntity[0]));
}
private static Set<Engine> getNerEngines(NerEntities.NerEntity... nerEntities) {
return Arrays.stream(nerEntities)
.map(NerEntities.NerEntity::engine)
.map(NerEntities::mapToPrimaryEngine)
.collect(Collectors.toSet());
}
private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
entityToDuplicate.addTextRange(newTextRange);

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -6,6 +6,7 @@ import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfi
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
@ -31,7 +32,6 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import com.iqser.red.storage.commons.exception.StorageException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.llm.service.LlmNerEntities;
@ -137,13 +137,13 @@ public class RedactionStorageService {
}
@SneakyThrows
public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) {
componentLogMongoService.saveComponentLog(dossierId, fileId, componentLog);
}
@SneakyThrows
public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {
@ -373,7 +373,12 @@ public class RedactionStorageService {
public LlmNerEntities getLlmNerEntities(String dossierId, String fileId) {
return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES), LlmNerEntities.class);
String objectId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES);
if (storageService.objectExists(TenantContext.getTenantId(), objectId)) {
return storageService.readJSONObject(TenantContext.getTenantId(), objectId, LlmNerEntities.class);
} else {
return LlmNerEntities.builder().entities(new ArrayList<>()).build();
}
}