Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-10615' into 'master'

Browse Source 
RED-10615: Full Analysis loop in DM

Closes RED-10615

See merge request redactmanager/persistence-service!898
This commit is contained in:
Maverick Studer 2024-12-05 11:54:56 +01:00
commit 08671583fe
2 changed files with 52 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java
View File

@ -25,6 +25,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
@ -50,6 +51,7 @@ public class ReanalysisRequiredStatusService {
DossierPersistenceService dossierPersistenceService; DossierPersistenceService dossierPersistenceService;
LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
ComponentMappingService componentMappingService; ComponentMappingService componentMappingService;
FileManagementServiceSettings settings;
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) { public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
@ -65,7 +67,12 @@ public class ReanalysisRequiredStatusService {
Map<String, DossierEntity> dossierMap = new HashMap<>(); Map<String, DossierEntity> dossierMap = new HashMap<>();
Map<String, Map<String, Integer>> componentMappingVersionMap = new HashMap<>(); Map<String, Map<String, Integer>> componentMappingVersionMap = new HashMap<>();
fileModels.forEach(entry -> { fileModels.forEach(entry -> {
var analysisRequiredResult = computeAnalysisRequired(entry, ignoreProcessingStates, dossierTemplateVersionMap, dossierVersionMap, dossierMap, componentMappingVersionMap); var analysisRequiredResult = computeAnalysisRequired(entry,
ignoreProcessingStates,
dossierTemplateVersionMap,
dossierVersionMap,
dossierMap,
componentMappingVersionMap);
entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired()); entry.setReanalysisRequired(analysisRequiredResult.isReanalysisRequired());
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired()); entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
entry.setComponentReanalysisRequired(analysisRequiredResult.isComponentReanalysisRequired()); entry.setComponentReanalysisRequired(analysisRequiredResult.isComponentReanalysisRequired());
@ -123,7 +130,7 @@ public class ReanalysisRequiredStatusService {
if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) { if (fileStatus.getLastFileAttributeChange() != null && fileStatus.getLastProcessed().isBefore(fileStatus.getLastFileAttributeChange())) {
return new AnalysisRequiredResult(true, false); return new AnalysisRequiredResult(true, false);
} else { } else {
return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap,componentMappingVersionMap); return requiresReanalysisBasedOnVersionDifference(fileStatus, dossier, dossierTemplateVersionMap, dossierVersionMap, componentMappingVersionMap);
} }
default: default:
return new AnalysisRequiredResult(false, false); return new AnalysisRequiredResult(false, false);
@ -134,14 +141,18 @@ public class ReanalysisRequiredStatusService {
} }
private AnalysisRequiredResult computeAnalysisRequiredForErrorState(DossierEntity dossier,FileModel fileStatus, Map<String, Map<VersionType, Long>> dossierTemplateVersionMap){ private AnalysisRequiredResult computeAnalysisRequiredForErrorState(DossierEntity dossier,
FileModel fileStatus,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap) {
Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
k -> buildVersionData(dossier.getDossierTemplateId())); k -> buildVersionData(dossier.getDossierTemplateId()));
var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L); var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L);
var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L); var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L);
var isRuleExecutionTimeout = fileStatus.getFileErrorInfo() != null && fileStatus.getFileErrorInfo().getErrorCode() != null && fileStatus.getFileErrorInfo().getErrorCode().equals( var isRuleExecutionTimeout = fileStatus.getFileErrorInfo() != null && fileStatus.getFileErrorInfo().getErrorCode() != null && fileStatus.getFileErrorInfo()
ErrorCode.RULES_EXECUTION_TIMEOUT); .getErrorCode()
.equals(ErrorCode.RULES_EXECUTION_TIMEOUT);
var fullAnalysisRequired = (!rulesVersionMatches || !componentRulesVersionMatches) && !isRuleExecutionTimeout; var fullAnalysisRequired = (!rulesVersionMatches || !componentRulesVersionMatches) && !isRuleExecutionTimeout;
@ -153,7 +164,7 @@ public class ReanalysisRequiredStatusService {
DossierEntity dossier, DossierEntity dossier,
Map<String, Map<VersionType, Long>> dossierTemplateVersionMap, Map<String, Map<VersionType, Long>> dossierTemplateVersionMap,
Map<String, Long> dossierVersionMap, Map<String, Long> dossierVersionMap,
Map<String, Map<String,Integer>> componentMappingVersionMap) { Map<String, Map<String, Integer>> componentMappingVersionMap) {
// get relevant versions // get relevant versions
Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
@ -162,7 +173,8 @@ public class ReanalysisRequiredStatusService {
Map<String, Integer> componentMappingVersions = componentMappingVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), Map<String, Integer> componentMappingVersions = componentMappingVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
k -> componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId()) k -> componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
.stream() .stream()
.collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion))); .collect(Collectors.toMap(ComponentMappingMetadata::getName,
ComponentMappingMetadata::getVersion)));
Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
@ -177,22 +189,23 @@ public class ReanalysisRequiredStatusService {
var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion; var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion;
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch; var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch;
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches || !aiVersionMatches; var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches || (settings.isLlmNerServiceEnabled() && !aiVersionMatches);
var componentReanalysisRequired = !dateFormatsVersionMatches; var componentReanalysisRequired = !dateFormatsVersionMatches;
if (reanalysisRequired || fullAnalysisRequired || componentReanalysisRequired) { if (reanalysisRequired || fullAnalysisRequired || componentReanalysisRequired) {
log.debug(buildMessage(fileStatus, log.debug(buildMessage(fileStatus,
rulesVersionMatches, rulesVersionMatches,
dossierTemplateVersions, dossierTemplateVersions,
componentRulesVersionMatches, componentRulesVersionMatches,
dateFormatsVersionMatches, dateFormatsVersionMatches,
dictionaryVersionMatches, dictionaryVersionMatches,
legalBasisVersionMatches, legalBasisVersionMatches,
dossierDictionaryVersionMatches, dossierDictionaryVersionMatches,
dossierDictionaryVersion, dossierDictionaryVersion,
mappingVersionAllMatch, mappingVersionAllMatch,
componentMappingVersions)); componentMappingVersions,
aiVersionMatches));
} }
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired, componentReanalysisRequired); return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired, componentReanalysisRequired);
@ -202,14 +215,15 @@ public class ReanalysisRequiredStatusService {
private String buildMessage(FileModel fileStatus, private String buildMessage(FileModel fileStatus,
boolean rulesVersionMatches, boolean rulesVersionMatches,
Map<VersionType, Long> dossierTemplateVersions, Map<VersionType, Long> dossierTemplateVersions,
boolean versionMatches,
boolean componentRulesVersionMatches, boolean componentRulesVersionMatches,
boolean dateFormatsVersionMatches,
boolean dictionaryVersionMatches, boolean dictionaryVersionMatches,
boolean legalBasisVersionMatches, boolean legalBasisVersionMatches,
boolean dossierDictionaryVersionMatches, boolean dossierDictionaryVersionMatches,
Long dossierDictionaryVersion, Long dossierDictionaryVersion,
boolean mappingVersionAllMatch, boolean mappingVersionAllMatch,
Map<String, Integer> componentMappingVersions) { Map<String, Integer> componentMappingVersions,
boolean aiVersionMatches) {
StringBuilder messageBuilder = new StringBuilder(); StringBuilder messageBuilder = new StringBuilder();
messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> "); messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> ");
@ -264,6 +278,21 @@ public class ReanalysisRequiredStatusService {
needComma = true; needComma = true;
} }
if (!dateFormatsVersionMatches) {
if (needComma) {
messageBuilder.append(", ");
}
messageBuilder.append("dateFormatsVersions: ").append(fileStatus.getDateFormatsVersion()).append("/").append(dossierTemplateVersions.getOrDefault(DATE_FORMATS, -1L));
needComma = true;
}
if (settings.isLlmNerServiceEnabled() && !aiVersionMatches) {
if (needComma) {
messageBuilder.append(", ");
}
messageBuilder.append("aiVersions: ").append(fileStatus.getAiCreationVersion()).append("/").append(dossierTemplateVersions.getOrDefault(AI_CREATION, 0L));
}
return messageBuilder.toString(); return messageBuilder.toString();
} }

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -737,11 +737,14 @@ public class FileStatusPersistenceService {
fileRepository.setAiCreationVersion(fileId, aiCreationVersion); fileRepository.setAiCreationVersion(fileId, aiCreationVersion);
} }
public List<FileEntity> findAllByIds(Set<String> fileIds) { public List<FileEntity> findAllByIds(Set<String> fileIds) {
return fileRepository.findAllById(fileIds); return fileRepository.findAllById(fileIds);
} }
public List<FileEntity> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) { public List<FileEntity> findAllDossierIdAndIds(String dossierId, Set<String> fileIds) {
return fileRepository.findAllDossierIdAndIds(dossierId, fileIds); return fileRepository.findAllDossierIdAndIds(dossierId, fileIds);