Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-10615' into 'master'

Browse Source 
RED-10615: Full Analysis loop in DM

Closes RED-10615

See merge request redactmanager/persistence-service!898
This commit is contained in:
Maverick Studer 2024-12-05 11:54:56 +01:00
commit 08671583fe
2 changed files with 52 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java
View File

@ -25,6 +25,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
@ -50,6 +51,7 @@ public class ReanalysisRequiredStatusService {
DossierPersistenceService dossierPersistenceService;
LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
ComponentMappingService componentMappingService;
FileManagementServiceSettings settings;
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
@ -65,7 +67,12 @@ public class ReanalysisRequiredStatusService {
Map<String, DossierEntity> dossierMap = new HashMap<>();
Map<String, Map<String, Integer>> componentMappingVersionMap = new HashMap<>();
fileModels.forEach(entry -> {
var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap, componentMappingVersionMap);
var analysisRequiredResult = computeAnalysisRequired(entry,
ignoreProcessingStates,
dossierTemplateVersionMap,
dossierVersionMap,
dossierMap,
componentMappingVersionMap);
entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
entry.setComponentReanalysisRequired(analysisRequiredResult.isComponentReanalysisRequired());
@ -123,7 +130,7 @@ public class ReanalysisRequiredStatusService {
if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) {
return new AnalysisRequiredResult(true, false);
} else {
return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap,componentMappingVersionMap);
return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap, componentMappingVersionMap);
}
default:
return new AnalysisRequiredResult(false, false);
@ -134,14 +141,18 @@ public class ReanalysisRequiredStatusService {
}
private AnalysisRequiredResult computeAnalysisRequiredForErrorState(DossierEntity dossier,FileModel fileStatus, Map<String, Map<VersionType, Long>> dossierTemplateVersionMap){
private AnalysisRequiredResult computeAnalysisRequiredForErrorState(DossierEntity dossier,
FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap) {
Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
k -> buildVersionData(dossier.getDossierTemplateId()));
var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L);
var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L);
var isRuleExecutionTimeout = fileStatus.getFileErrorInfo() != null && fileStatus.getFileErrorInfo().getErrorCode() != null && fileStatus.getFileErrorInfo().getErrorCode().equals(
ErrorCode.RULES_EXECUTION_TIMEOUT);
var isRuleExecutionTimeout = fileStatus.getFileErrorInfo() != null && fileStatus.getFileErrorInfo().getErrorCode() != null && fileStatus.getFileErrorInfo()
.getErrorCode()
.equals(ErrorCode.RULES_EXECUTION_TIMEOUT);
var fullAnalysisRequired = (!rulesVersionMatches || !componentRulesVersionMatches) && !isRuleExecutionTimeout;
@ -153,7 +164,7 @@ public class ReanalysisRequiredStatusService {
DossierEntity dossier,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap,
Map<String, Map<String,Integer>> componentMappingVersionMap) {
Map<String, Map<String, Integer>> componentMappingVersionMap) {
// get relevant versions
Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
@ -162,7 +173,8 @@ public class ReanalysisRequiredStatusService {
Map<String, Integer> componentMappingVersions = componentMappingVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
k -> componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
.stream()
.collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion)));
.collect(Collectors.toMap(ComponentMappingMetadata::getName,
ComponentMappingMetadata::getVersion)));
Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
@ -177,22 +189,23 @@ public class ReanalysisRequiredStatusService {
var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion;
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch;
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches || !aiVersionMatches;
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches || (settings.isLlmNerServiceEnabled() && !aiVersionMatches);
var componentReanalysisRequired = !dateFormatsVersionMatches;
if (reanalysisRequired || fullAnalysisRequired || componentReanalysisRequired) {
log.debug(buildMessage(fileStatus,
rulesVersionMatches,
dossierTemplateVersions,
componentRulesVersionMatches,
dateFormatsVersionMatches,
dictionaryVersionMatches,
legalBasisVersionMatches,
dossierDictionaryVersionMatches,
dossierDictionaryVersion,
mappingVersionAllMatch,
componentMappingVersions));
rulesVersionMatches,
dossierTemplateVersions,
componentRulesVersionMatches,
dateFormatsVersionMatches,
dictionaryVersionMatches,
legalBasisVersionMatches,
dossierDictionaryVersionMatches,
dossierDictionaryVersion,
mappingVersionAllMatch,
componentMappingVersions,
aiVersionMatches));
}
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired, componentReanalysisRequired);
@ -202,14 +215,15 @@ public class ReanalysisRequiredStatusService {
private String buildMessage(FileModel fileStatus,
boolean rulesVersionMatches,
Map<VersionType, Long> dossierTemplateVersions,
boolean versionMatches,
boolean componentRulesVersionMatches,
boolean dateFormatsVersionMatches,
boolean dictionaryVersionMatches,
boolean legalBasisVersionMatches,
boolean dossierDictionaryVersionMatches,
Long dossierDictionaryVersion,
boolean mappingVersionAllMatch,
Map<String, Integer> componentMappingVersions) {
Map<String, Integer> componentMappingVersions,
boolean aiVersionMatches) {
StringBuilder messageBuilder = new StringBuilder();
messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> ");
@ -264,6 +278,21 @@ public class ReanalysisRequiredStatusService {
needComma = true;
}
if (!dateFormatsVersionMatches) {
if (needComma) {
messageBuilder.append(", ");
}
messageBuilder.append("dateFormatsVersions: ").append(fileStatus.getDateFormatsVersion()).append("/").append(dossierTemplateVersions.getOrDefault(DATE_FORMATS, -1L));
needComma = true;
}
if (settings.isLlmNerServiceEnabled() && !aiVersionMatches) {
if (needComma) {
messageBuilder.append(", ");
}
messageBuilder.append("aiVersions: ").append(fileStatus.getAiCreationVersion()).append("/").append(dossierTemplateVersions.getOrDefault(AI_CREATION, 0L));
}
return messageBuilder.toString();
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -737,11 +737,14 @@ public class FileStatusPersistenceService {
fileRepository.setAiCreationVersion(fileId, aiCreationVersion);
}
public List<FileEntity> findAllByIds(Set<String> fileIds) {
return fileRepository.findAllById(fileIds);
}
public List<FileEntity> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
return fileRepository.findAllDossierIdAndIds(dossierId, fileIds);