Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-10072: AI description field and toggle for entities

Browse Source
This commit is contained in:
Maverick Studer 2024-11-07 14:43:54 +01:00
parent 584473565f
commit 31c3ce45f0
26 changed files with 237 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
persistence-service-v1/persistence-service-processor-v1/build.gradle.kts
View File

@ -39,7 +39,7 @@ dependencies {
exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1")
}
api("com.knecon.fforesight:azure-ocr-service-api:0.13.0")
implementation("com.knecon.fforesight:llm-service-api:1.17.0")
implementation("com.knecon.fforesight:llm-service-api:1.20.0-RED10072.2")
api("com.knecon.fforesight:jobs-commons:0.10.0")
api("com.iqser.red.commons:storage-commons:2.50.0")
api("com.knecon.fforesight:tenant-commons:0.31.0") {

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/EntityTypeImportService.java
View File

@ -125,7 +125,7 @@ public class EntityTypeImportService {
returnedType.getType(),
DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryPersistenceService.setVersion(returnedType.getTypeId(), type.getVersion());
dictionaryPersistenceService.setVersions(returnedType.getTypeId(), type.getVersion(), type.getAiCreationVersion());
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java
View File

@ -49,8 +49,14 @@ public class TypeEntity {
@Column(length = 4000)
private String description;
@Column
private boolean aiCreationEnabled;
@Column(length = 4000)
private String aiDescription;
@Column
private long version;
@Column
private long aiCreationVersion;
@Column
private boolean addToDictionaryAction;
@Column
private boolean hasDictionary;

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -110,6 +110,9 @@ public class FileEntity {
@Column
private long dictionaryVersion;
@Column
private long aiCreationVersion;
@Column
private long rulesVersion;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/migration/migrations/AddGraphicDictionaryType19.java
View File

@ -60,6 +60,8 @@ public class AddGraphicDictionaryType19 extends Migration {
false,
"Empty dictionary used to configure graphic colors.",
false,
null,
false,
"Graphic",
null,
true,

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/model/AnalysisType.java
View File

@ -4,6 +4,7 @@ public enum AnalysisType {
DEFAULT,
MANUAL_REDACTION_REANALYZE,
FORCE_ANALYSE,
COMPONENTS_ONLY_REANALYZE

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java
View File

@ -100,6 +100,8 @@ public class DictionaryManagementService {
typeRequest.isCaseInsensitive(),
typeRequest.isRecommendation(),
typeRequest.getDescription(),
typeRequest.isAiCreationEnabled(),
typeRequest.getAiDescription(),
typeRequest.isAddToDictionaryAction(),
typeRequest.getLabel(),
typeRequest.getDossierId(),

14
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java
View File

@ -178,6 +178,8 @@ public class DictionaryService {
.isCaseInsensitive(typeValue.isCaseInsensitive())
.isRecommendation(typeValue.isRecommendation())
.description(typeValue.getDescription())
.aiCreationEnabled(typeValue.isAiCreationEnabled())
.aiDescription(typeValue.getAiDescription())
.addToDictionaryAction(typeValue.isAddToDictionaryAction())
.label(typeValue.getLabel())
.hasDictionary(typeValue.isHasDictionary())
@ -201,6 +203,8 @@ public class DictionaryService {
.isCaseInsensitive(typeValue.isCaseInsensitive())
.isRecommendation(typeValue.isRecommendation())
.description(typeValue.getDescription())
.aiCreationEnabled(typeValue.isAiCreationEnabled())
.aiDescription(typeValue.getAiDescription())
.addToDictionaryAction(typeEntity.isAddToDictionaryAction())
.label(typeValue.getLabel())
.hasDictionary(typeValue.isHasDictionary())
@ -233,6 +237,8 @@ public class DictionaryService {
.isCaseInsensitive(typeValue.isCaseInsensitive())
.isRecommendation(typeValue.isRecommendation())
.description(typeValue.getDescription())
.aiCreationEnabled(typeValue.isAiCreationEnabled())
.aiDescription(typeValue.getAiDescription())
.addToDictionaryAction(typeValue.isAddToDictionaryAction())
.label(typeValue.getLabel())
.hasDictionary(typeValue.isHasDictionary())
@ -297,6 +303,8 @@ public class DictionaryService {
.caseInsensitive(typeResult.isCaseInsensitive())
.recommendation(typeResult.isRecommendation())
.description(typeResult.getDescription())
.aiCreationEnabled(typeResult.isAiCreationEnabled())
.aiDescription(typeResult.getAiDescription())
.addToDictionaryAction(typeResult.isAddToDictionaryAction())
.label(typeResult.getLabel())
.hasDictionary(typeResult.isHasDictionary())
@ -488,11 +496,7 @@ public class DictionaryService {
@PreAuthorize("hasAuthority('" + ADD_UPDATE_DICTIONARY_TYPE + "')")
public void changeAddToDictionary(String type, String dossierTemplateId, String dossierId, boolean addToDictionary) {
var typeEntity = dictionaryPersistenceService.getType(toTypeId(type, dossierTemplateId, dossierId));
if (typeEntity.isDossierDictionaryOnly()) {
typeEntity.setAddToDictionaryAction(addToDictionary);
dictionaryPersistenceService.saveType(typeEntity);
}
dictionaryPersistenceService.updateAddToDictionary(toTypeId(type, dossierTemplateId, dossierId), addToDictionary);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java
View File

@ -237,6 +237,8 @@ public class DossierTemplateCloneService {
t.isCaseInsensitive(),
t.isRecommendation(),
t.getDescription(),
t.isAiCreationEnabled(),
t.getAiDescription(),
t.isAddToDictionaryAction(),
t.getLabel(),
null,

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusMapper.java
View File

@ -43,6 +43,7 @@ public class FileStatusMapper {
.hasAnnotationComments(status.isHasAnnotationComments())
.uploader(status.getUploader())
.dictionaryVersion(status.getDictionaryVersion())
.aiCreationVersion(status.getAiCreationVersion())
.rulesVersion(status.getRulesVersion())
.componentRulesVersion(status.getComponentRulesVersion())
.dateFormatsVersion(status.getDateFormatsVersion())

71
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -20,6 +20,7 @@ import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ApplicationType;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ComponentDefinitionEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -41,6 +42,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.layoutp
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.QueueMessageIdentifierService;
import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ComponentDefinitionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
@ -74,6 +76,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.Com
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
import com.knecon.fforesight.llm.service.EntityAiDescription;
import com.knecon.fforesight.llm.service.LlmNerMessage;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
@ -121,6 +124,7 @@ public class FileStatusService {
ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
EntityLogMongoService entityLogMongoService;
ComponentLogMongoService componentLogMongoService;
DictionaryPersistenceService dictionaryPersistenceService;
WebsocketService websocketService;
@ -353,11 +357,29 @@ public class FileStatusService {
return;
}
if (settings.isLlmNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.LLM_NER_ENTITIES)) {
log.info("Add file: {} from dossier {} to LLM NER queue", fileId, dossierId);
addToLLMNerQueue(dossierId, fileId);
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
boolean forceAnalysis = false;
if (settings.isLlmNerServiceEnabled()) {
boolean objectExists = fileManagementStorageService.objectExists(dossierId, fileId, FileType.LLM_NER_ENTITIES);
long currentVersion = fileModel.getAiCreationVersion();
long expectedVersion = toAiCreationVersion(dictionaryPersistenceService.getTypesForAiCreation(dossierTemplate.getId()));
boolean versionsMatch = currentVersion == expectedVersion;
if (!objectExists || !versionsMatch) {
log.info("Adding file: {} from dossier {} to LLM NER queue", fileId, dossierId);
if (addToLLMNerQueue(dossierTemplate.getId(), dossierId, fileId)) {
sendReadOnlyAnalysisEvent(dossierId, fileId, fileEntity);
return;
} else {
log.info("No entities with enabled AI creation are configured.");
if (objectExists) {
forceAnalysis = true;
fileStatusPersistenceService.setAiCreationVersion(fileId, expectedVersion);
fileManagementStorageService.deleteObject(dossierId, fileId, FileType.LLM_NER_ENTITIES);
log.info("Deleted old LLM NER entities file due to ai creation version change.");
}
}
}
}
if (settings.isAzureNerServiceEnabled() && !fileManagementStorageService.objectExists(dossierId, fileId, FileType.AZURE_NER_ENTITIES)) {
@ -369,6 +391,9 @@ public class FileStatusService {
boolean reanalyse = fileModel.isReanalysisRequired() || analysisType.equals(AnalysisType.MANUAL_REDACTION_REANALYZE);
MessageType messageType = calculateMessageType(reanalyse, fileModel.getProcessingStatus(), fileModel);
if(analysisType == AnalysisType.FORCE_ANALYSE || forceAnalysis) {
messageType = MessageType.ANALYSE;
}
var analyseRequest = AnalyzeRequest.builder()
.messageType(messageType)
@ -581,7 +606,17 @@ public class FileStatusService {
}
protected void addToLLMNerQueue(String dossierId, String fileId) {
protected boolean addToLLMNerQueue(String dossierTemplateId, String dossierId, String fileId) {
List<TypeEntity> relevantTypesForAiCreation = dictionaryPersistenceService.getTypesForAiCreation(dossierTemplateId);
long aiCreationVersion = toAiCreationVersion(relevantTypesForAiCreation);
List<EntityAiDescription> entityAiDescriptions = relevantTypesForAiCreation.stream()
.map(te -> new EntityAiDescription(te.getType(), te.getAiDescription()))
.toList();
if (entityAiDescriptions.isEmpty()) {
return false;
}
setStatusNerAnalyzing(fileId);
boolean protoFilesExist = fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
@ -602,21 +637,31 @@ public class FileStatusService {
TenantContext.getTenantId(),
LlmNerMessage.builder()
.identifier(QueueMessageIdentifierService.buildIdentifier(dossierId, fileId, false))
.entityAiDescriptions(entityAiDescriptions)
.chunksStorageId(chunksStorageId)
.documentPagesStorageId(documentPagesStorageId)
.documentStructureStorageId(documentStructureStorageId)
.documentTextStorageId(documentTextStorageId)
.documentPositionStorageId(documentPositionStorageId)
.resultStorageId(resultStorageId)
.aiCreationVersion(aiCreationVersion)
.build(),
message -> {
message.getMessageProperties().setPriority(1);
return message;
});
return true;
}
private static long toAiCreationVersion(List<TypeEntity> relevantTypesForAiCreation) {
return relevantTypesForAiCreation.stream()
.mapToLong(TypeEntity::getAiCreationVersion).sum();
}
protected void addToAzureNerQueue(String dossierTemplateId, String dossierId, String fileId) {
setStatusNerAnalyzing(fileId);
@ -1019,6 +1064,20 @@ public class FileStatusService {
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.DEFAULT);
}
@Transactional
public void setStatusForceAnalyse(String dossierId, String fileId, boolean priority) {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
if (fileStatus.isExcluded()) {
log.debug("File {} is excluded", fileStatus.getId());
return;
}
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE);
addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), AnalysisType.FORCE_ANALYSE);
}
public void updateLayoutProcessedTime(String fileId) {

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import static com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService.VersionType.AI_CREATION;
import static com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService.VersionType.COMPONENT_RULES;
import static com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService.VersionType.DATE_FORMATS;
import static com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisRequiredStatusService.VersionType.DICTIONARY;
@ -16,6 +17,7 @@ import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DateFormatsPersistenceService;
@ -155,10 +157,11 @@ public class ReanalysisRequiredStatusService {
var dateFormatsVersionMatches = fileStatus.getDateFormatsVersion() == dossierTemplateVersions.getOrDefault(DATE_FORMATS, -1L);
var dictionaryVersionMatches = fileStatus.getDictionaryVersion() == dossierTemplateVersions.getOrDefault(DICTIONARY, -1L);
var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L);
var aiVersionMatches = fileStatus.getAiCreationVersion() == dossierTemplateVersions.getOrDefault(AI_CREATION, 0L);
var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion;
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch;
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches;
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches || !aiVersionMatches;
var componentReanalysisRequired = !dateFormatsVersionMatches;
if (reanalysisRequired || fullAnalysisRequired || componentReanalysisRequired) {
@ -287,6 +290,10 @@ public class ReanalysisRequiredStatusService {
versions.put(DATE_FORMATS, dateFormatsPersistenceService.getVersion(dossierTemplateId));
versions.put(DICTIONARY, dictionaryPersistenceService.getVersion(dossierTemplateId));
versions.put(LEGAL_BASIS, legalBasisMappingPersistenceService.getVersion(dossierTemplateId));
versions.put(AI_CREATION,
dictionaryPersistenceService.getTypesForAiCreation(dossierTemplateId)
.stream()
.mapToLong(TypeEntity::getAiCreationVersion).sum());
return versions;
}
@ -314,6 +321,7 @@ public class ReanalysisRequiredStatusService {
RULES,
COMPONENT_RULES,
DATE_FORMATS,
AI_CREATION,
DICTIONARY,
LEGAL_BASIS
}

72
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -36,6 +37,12 @@ public class DictionaryPersistenceService {
private final EntryRepository entryRepository;
public List<TypeEntity> getTypesForDossier(String dossierId) {
return typeRepository.findByDossierId(dossierId);
}
@Transactional(Transactional.TxType.REQUIRES_NEW) // This needs a single transaction because of the creation of types on the fly, otherwise add entries will fails
public TypeEntity addType(String type,
String dossierTemplateId,
@ -47,6 +54,8 @@ public class DictionaryPersistenceService {
boolean caseInsensitive,
boolean isRecommendation,
String description,
boolean aiCreationEnabled,
String aiDescription,
boolean addToDictionaryAction,
String label,
String dossierId,
@ -68,12 +77,15 @@ public class DictionaryPersistenceService {
.skippedHexColor(skippedHexColor)
.rank(rank)
.description(description)
.aiCreationEnabled(aiCreationEnabled)
.aiDescription(aiDescription)
.isHint(isHint)
.isCaseInsensitive(caseInsensitive)
.isRecommendation(isRecommendation)
.addToDictionaryAction(addToDictionaryAction)
.label(label)
.version(1)
.aiCreationVersion(0)
.hasDictionary(hasDictionary)
.systemManaged(systemManaged)
.autoHideSkipped(autoHideSkipped)
@ -82,6 +94,10 @@ public class DictionaryPersistenceService {
.experimental(experimental)
.build();
if (aiCreationEnabled && aiDescription != null && !aiDescription.isBlank()) {
t.setAiCreationVersion(1);
}
return typeRepository.saveAndFlush(t);
}
@ -121,6 +137,7 @@ public class DictionaryPersistenceService {
// }
type.setVersion(type.getVersion() + 1);
incrementAiCreationVersionIfNeeded(type, typeValueRequest);
checkRankAlreadyExists(type.getType(),
type.getDossierTemplate().getId(),
typeValueRequest.getRank(),
@ -130,6 +147,8 @@ public class DictionaryPersistenceService {
type.setRecommendationHexColor(typeValueRequest.getRecommendationHexColor());
type.setSkippedHexColor(typeValueRequest.getSkippedHexColor());
type.setDescription(typeValueRequest.getDescription());
type.setAiCreationEnabled(typeValueRequest.isAiCreationEnabled());
type.setAiDescription(typeValueRequest.getAiDescription());
type.setLabel(typeValueRequest.getLabel());
type.setAddToDictionaryAction(typeValueRequest.isAddToDictionaryAction());
} else {
@ -145,6 +164,7 @@ public class DictionaryPersistenceService {
"dossier",
"id",
"version",
"aiCreationVersion",
"dossierDictionaryOnly");
}
typeRepository.saveAndFlush(type);
@ -152,6 +172,15 @@ public class DictionaryPersistenceService {
}
private void incrementAiCreationVersionIfNeeded(TypeEntity currentType, TypeEntity newType) {
if ((newType.isAiCreationEnabled() && !Objects.equals(currentType.getAiDescription(), newType.getAiDescription())) || //
(currentType.isAiCreationEnabled() != newType.isAiCreationEnabled() && newType.getAiDescription() != null && !newType.getAiDescription().isBlank())) {
currentType.setAiCreationVersion(currentType.getAiCreationVersion() + 1);
}
}
public List<TypeEntity> getCumulatedTypes(String dossierTemplateId, String dossierId, boolean includeDeleted) {
var types = typeRepository.findAllTypesByDossierTemplateIdOrDossierId(dossierTemplateId, dossierId);
@ -236,10 +265,11 @@ public class DictionaryPersistenceService {
typeRepository.updateByIdSetIncrementVersionByOne(typeId);
}
@Transactional
public void setVersion(String typeId, long version) {
typeRepository.updateVersionForType(typeId, version);
@Transactional
public void setVersions(String typeId, long version, long aiCreationVersion) {
typeRepository.updateVersionsForType(typeId, version, aiCreationVersion);
}
@ -255,6 +285,15 @@ public class DictionaryPersistenceService {
}
public List<TypeEntity> getTypesForAiCreation(String dossierTemplateId) {
return getAllTypesForDossierTemplate(dossierTemplateId, false).stream()
.filter(TypeEntity::isAiCreationEnabled)
.filter(typeEntity -> typeEntity.getAiDescription() != null && !typeEntity.getAiDescription().isBlank())
.toList();
}
public List<DictionarySummaryResponse> getDictionarySummaryForDossierTemplateId(Set<String> dossierTemplateIds) {
return typeRepository.findDictionarySummaryList(dossierTemplateIds);
@ -279,12 +318,15 @@ public class DictionaryPersistenceService {
.hexColor(typeEntity.get().getHexColor())
.rank(typeEntity.get().getRank())
.description(typeEntity.get().getDescription())
.aiCreationEnabled(typeEntity.get().isAiCreationEnabled())
.aiDescription(typeEntity.get().getAiDescription())
.isHint(typeEntity.get().isHint())
.isCaseInsensitive(typeEntity.get().isCaseInsensitive())
.isRecommendation(typeEntity.get().isRecommendation())
.addToDictionaryAction(typeEntity.get().isAddToDictionaryAction())
.label(typeEntity.get().getLabel())
.version(typeEntity.get().getVersion() + 1)
.aiCreationVersion(typeEntity.get().getAiCreationVersion() + 1)
.build());
entryRepository.updateTypeIdAndIncrementVersionByOne(typeId, newTypeId);
@ -292,18 +334,6 @@ public class DictionaryPersistenceService {
}
public void saveAllTypes(List<TypeEntity> types) {
typeRepository.saveAll(types);
}
public void saveType(TypeEntity type) {
typeRepository.saveAndFlush(type);
}
@Transactional
public int undeleteType(String typeId) {
@ -325,6 +355,18 @@ public class DictionaryPersistenceService {
}
@Transactional
public void updateAddToDictionary(String typeId, boolean addToDictionary) {
var typeEntity = getType(typeId);
if (typeEntity.isDossierDictionaryOnly()) {
typeEntity.setAddToDictionaryAction(addToDictionary);
typeRepository.saveAndFlush(typeEntity);
}
}
@Transactional
public void setExperimentalTrue(String typeId) {

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -732,6 +732,11 @@ public class FileStatusPersistenceService {
}
@Transactional
public void setAiCreationVersion(String fileId, long aiCreationVersion) {
fileRepository.setAiCreationVersion(fileId, aiCreationVersion);
}
public List<FileEntity> findAllByIds(Set<String> fileIds) {
return fileRepository.findAllById(fileIds);

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -69,7 +69,8 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
f.lastUpdated = :lastUpdated, \
f.lastProcessed = :lastProcessed, \
f.processingErrorCounter = :processingErrorCounter \
where f.id = :fileId""")
where f.id = :fileId
""")
void updateProcessingStatus(@Param("fileId") String fileId,
@Param("numberOfPages") int numberOfPages,
@Param("processingStatus") ProcessingStatus processingStatus,
@ -87,6 +88,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Param("processingErrorCounter") int processingErrorCounter);
@Modifying
@Query("update FileEntity f set f.aiCreationVersion = :aiCreationVersion where f.id = :fileId")
void setAiCreationVersion(@Param("fileId") String fileId, @Param("aiCreationVersion") long aiCreationVersion);
@Modifying
@Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate,"
+ " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId")
@ -232,7 +238,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
+ "f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, "
+ "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, "
+ "f.redactionModificationDate = null, "
+ "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
+ "f.dictionaryVersion = 0, f.aiCreationVersion=0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
+ "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
+ "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, "
+ "f.processingErrorCounter = 0, f.errorCause = null, f.errorQueue = null, f.errorService = null, "
@ -253,7 +259,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
+ "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, "
+ " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastProcessed = null,"
+ "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
+ "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
+ "f.dictionaryVersion = 0, f.aiCreationVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
+ "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
+ "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0, "
+ "f.overwriteFileCounter = coalesce(f.overwriteFileCounter, 0) + 1, "

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java
View File

@ -97,7 +97,7 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
@Modifying
@Query("update TypeEntity t set t.version = :version where t.id = :typeId")
void updateVersionForType(@Param("typeId") String typeId, @Param("version") long version);
@Query("update TypeEntity t set t.version = :version, t.aiCreationVersion = :aiCreationVersion where t.id = :typeId")
void updateVersionsForType(@Param("typeId") String typeId, @Param("version") long version, @Param("aiCreationVersion") long aiCreationVersion);
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/queue/NerMessageReceiver.java
View File

@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.management.v1.processor.configuration.M
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusProcessingUpdateService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.layoutparsing.QueueMessageIdentifierService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import com.knecon.fforesight.llm.service.LlmNerMessage;
import com.knecon.fforesight.llm.service.LlmNerResponseMessage;
@ -35,6 +36,7 @@ public class NerMessageReceiver {
private final FileStatusService fileStatusService;
private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateService fileStatusProcessingUpdateService;
private final FileStatusPersistenceService fileStatusPersistenceService;
private final ObservationRegistry observationRegistry;
@ -63,7 +65,8 @@ public class NerMessageReceiver {
addFileIdToTrace(fileId);
log.info("Received message from {} for dossierId {} and fileId {}", LLM_ENTITY_RESPONSE_LISTENER_ID, dossierId, fileId);
fileStatusService.setStatusAnalyse(dossierId, fileId, false);
fileStatusPersistenceService.setAiCreationVersion(fileId, message.getAiCreationVersion());
fileStatusService.setStatusForceAnalyse(dossierId, fileId, false);
}

2
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -245,3 +245,5 @@ databaseChangeLog:
file: db/changelog/tenant/150-add-indexes-across-tables-for-performance.yaml
- include:
file: db/changelog/tenant/151-add-component-mapping-indexes.yaml
- include:
file: db/changelog/tenant/152-add-ai-fields-to-entity.yaml

31
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/152-add-ai-fields-to-entity.yaml Normal file
View File

@ -0,0 +1,31 @@
databaseChangeLog:
- changeSet:
id: add-ai-fields-to-entity
author: maverick
changes:
- addColumn:
columns:
- column:
name: ai_creation_enabled
type: BOOLEAN
defaultValue: false
- column:
name: ai_description
type: VARCHAR(4000)
- column:
name: ai_creation_version
type: INT
constraints:
nullable: false
defaultValue: 0
tableName: entity
- addColumn:
columns:
- column:
name: ai_creation_version
type: INT
constraints:
nullable: false
defaultValue: 0
tableName: file

6
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/CreateTypeValue.java
View File

@ -48,6 +48,12 @@ public class CreateTypeValue {
@Schema(description = "The description of the dictionary type")
private String description;
@Schema(description = "True if AI is to be used to create the type, default is false")
private boolean aiCreationEnabled;
@Schema(description = "The AI description of the dictionary type")
private String aiDescription;
@Schema(description = "If true the ui will add a action to add values to dictionary")
private boolean addToDictionaryAction;

2
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatus.java
View File

@ -96,6 +96,8 @@ public class FileStatus {
private String uploader;
@Schema(description = "Shows which dictionary version was used during the analysis.")
private long dictionaryVersion;
@Schema(description = "Shows which ai creation version was used during the analysis.")
private long aiCreationVersion;
@Schema(description = "Shows which entity rules version was used during the analysis.")
private long rulesVersion;
@Schema(description = "Shows which component rules version was used during the analysis.")

6
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/TypeValue.java
View File

@ -48,6 +48,12 @@ public class TypeValue {
@Schema(description = "The description of the dictionary type")
private String description;
@Schema(description = "True if the type is also to be created via AI, default is false")
private boolean aiCreationEnabled;
@Schema(description = "The AI description of the dictionary type")
private String aiDescription;
@Schema(description = "If true the ui will add a action to add values to dictionary")
private boolean addToDictionaryAction;

6
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/UpdateTypeValue.java
View File

@ -38,6 +38,12 @@ public class UpdateTypeValue {
@Schema(description = "The description of the dictionary type")
private String description;
@Schema(description = "True if AI is to be used to create the type, default is false")
private boolean aiCreationEnabled;
@Schema(description = "The AI description of the dictionary type")
private String aiDescription;
@Schema(description = "If true the ui will add a action to add values to dictionary")
private boolean addToDictionaryAction;

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/analysislog/entitylog/Engine.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model.analysislo
public enum Engine {
DICTIONARY,
NER,
LLM_NER,
RULE,
MANUAL,
IMPORTED,

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -47,6 +47,7 @@ public class FileModel {
private boolean componentReanalysisRequired;
private String uploader;
private long dictionaryVersion;
private long aiCreationVersion;
private long rulesVersion;
private long componentRulesVersion;
private long dateFormatsVersion;

3
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/type/Type.java
View File

@ -30,7 +30,10 @@ public class Type {
private boolean isCaseInsensitive;
private boolean isRecommendation;
private String description;
private boolean aiCreationEnabled;
private String aiDescription;
private long version;
private long aiCreationVersion;
private boolean addToDictionaryAction;
private boolean dossierDictionaryOnly;
private String dossierTemplateId;