RED-8670: integrate table inference from research
* introduce controllerAdvice to ControllerV2 package
This commit is contained in:
parent
d592c68321
commit
844466812b
@ -14,8 +14,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
|
|||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||||
|
|
||||||
import jakarta.persistence.CascadeType;
|
import jakarta.persistence.CascadeType;
|
||||||
|
import jakarta.persistence.CollectionTable;
|
||||||
import jakarta.persistence.Column;
|
import jakarta.persistence.Column;
|
||||||
import jakarta.persistence.Convert;
|
import jakarta.persistence.Convert;
|
||||||
|
import jakarta.persistence.ElementCollection;
|
||||||
import jakarta.persistence.Entity;
|
import jakarta.persistence.Entity;
|
||||||
import jakarta.persistence.EnumType;
|
import jakarta.persistence.EnumType;
|
||||||
import jakarta.persistence.Enumerated;
|
import jakarta.persistence.Enumerated;
|
||||||
@ -197,6 +199,9 @@ public class FileEntity {
|
|||||||
@Column
|
@Column
|
||||||
private OffsetDateTime errorTimestamp;
|
private OffsetDateTime errorTimestamp;
|
||||||
|
|
||||||
|
@ElementCollection(fetch = FetchType.EAGER)
|
||||||
|
private List<FileEntityComponentMappingVersionEntity> componentMappingVersions;
|
||||||
|
|
||||||
|
|
||||||
public OffsetDateTime getLastOCRTime() {
|
public OffsetDateTime getLastOCRTime() {
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,17 @@
|
|||||||
|
package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
|
||||||
|
|
||||||
|
import jakarta.persistence.Embeddable;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
@Embeddable
|
||||||
|
@AllArgsConstructor
|
||||||
|
@NoArgsConstructor
|
||||||
|
public class FileEntityComponentMappingVersionEntity {
|
||||||
|
|
||||||
|
private String name;
|
||||||
|
private Integer version;
|
||||||
|
|
||||||
|
}
|
||||||
@ -17,7 +17,7 @@ public interface ComponentMappingEntityMapper {
|
|||||||
ComponentMappingMetadata toComponentMappingMetaData(ComponentMappingEntity componentMappingMetaData);
|
ComponentMappingMetadata toComponentMappingMetaData(ComponentMappingEntity componentMappingMetaData);
|
||||||
|
|
||||||
|
|
||||||
List<ComponentMappingMetadata> toComponentMappingMetaDataList(List<ComponentMappingEntity> componentMappingMetaData);
|
List<ComponentMappingMetadata> toComponentMappingMetaDataList(List<ComponentMappingEntity> componentMappingEntities);
|
||||||
|
|
||||||
|
|
||||||
ComponentMappingEntity toComponentMappingEntity(ComponentMappingMetadata componentMappingSummary);
|
ComponentMappingEntity toComponentMappingEntity(ComponentMappingMetadata componentMappingSummary);
|
||||||
|
|||||||
@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo
|
|||||||
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
|
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
|
import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@ -520,7 +521,8 @@ public class FileStatusService {
|
|||||||
analyzeResult.getDuration(),
|
analyzeResult.getDuration(),
|
||||||
analyzeResult.getDossierDictionaryVersion(),
|
analyzeResult.getDossierDictionaryVersion(),
|
||||||
analyzeResult.getAnalysisVersion(),
|
analyzeResult.getAnalysisVersion(),
|
||||||
analyzeResult.getAnalysisNumber());
|
analyzeResult.getAnalysisNumber(),
|
||||||
|
analyzeResult.getUsedComponentMappings());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -7,8 +7,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.service.
|
|||||||
|
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@ -19,23 +22,28 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
|||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||||
|
|
||||||
|
import lombok.AccessLevel;
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.experimental.FieldDefaults;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||||
public class ReanalysisRequiredStatusService {
|
public class ReanalysisRequiredStatusService {
|
||||||
|
|
||||||
private final DictionaryPersistenceService dictionaryPersistenceService;
|
DictionaryPersistenceService dictionaryPersistenceService;
|
||||||
private final RulesPersistenceService rulesPersistenceService;
|
RulesPersistenceService rulesPersistenceService;
|
||||||
private final DossierPersistenceService dossierPersistenceService;
|
DossierPersistenceService dossierPersistenceService;
|
||||||
private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
|
LegalBasisMappingPersistenceService legalBasisMappingPersistenceService;
|
||||||
|
ComponentMappingService componentMappingService;
|
||||||
|
|
||||||
|
|
||||||
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
|
public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) {
|
||||||
@ -118,39 +126,141 @@ public class ReanalysisRequiredStatusService {
|
|||||||
Map<String, Long> dossierVersionMap) {
|
Map<String, Long> dossierVersionMap) {
|
||||||
|
|
||||||
// get relevant versions
|
// get relevant versions
|
||||||
var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), k -> buildVersionData(dossier.getDossierTemplateId()));
|
Map<VersionType, Long> dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
|
||||||
var dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
|
k -> buildVersionData(dossier.getDossierTemplateId()));
|
||||||
|
|
||||||
|
Map<String, Integer> componentMappingVersions = componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId())
|
||||||
|
.stream()
|
||||||
|
.collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion));
|
||||||
|
|
||||||
|
Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId()));
|
||||||
|
|
||||||
// compute matches
|
// compute matches
|
||||||
|
var mappingVersionAllMatch = mappingVersionsEqual(fileStatus, componentMappingVersions);
|
||||||
var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L);
|
var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L);
|
||||||
var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L);
|
var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L);
|
||||||
var dictionaryVersionMatches = fileStatus.getDictionaryVersion() == dossierTemplateVersions.getOrDefault(DICTIONARY, -1L);
|
var dictionaryVersionMatches = fileStatus.getDictionaryVersion() == dossierTemplateVersions.getOrDefault(DICTIONARY, -1L);
|
||||||
var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L);
|
var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L);
|
||||||
var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion;
|
var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion;
|
||||||
|
|
||||||
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches;
|
var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch;
|
||||||
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches;
|
var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches;
|
||||||
|
|
||||||
if (reanalysisRequired || fullAnalysisRequired) {
|
if (reanalysisRequired || fullAnalysisRequired) {
|
||||||
log.info(
|
|
||||||
"For file: {}-{} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}",
|
log.info(buildMessage(fileStatus,
|
||||||
fileStatus.getId(),
|
rulesVersionMatches,
|
||||||
fileStatus.getFilename(),
|
dossierTemplateVersions,
|
||||||
fileStatus.getRulesVersion(),
|
componentRulesVersionMatches,
|
||||||
dossierTemplateVersions.getOrDefault(RULES, -1L),
|
dictionaryVersionMatches,
|
||||||
fileStatus.getComponentRulesVersion(),
|
legalBasisVersionMatches,
|
||||||
dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L),
|
dossierDictionaryVersionMatches,
|
||||||
fileStatus.getDictionaryVersion(),
|
dossierDictionaryVersion,
|
||||||
dossierTemplateVersions.getOrDefault(DICTIONARY, -1L),
|
mappingVersionAllMatch,
|
||||||
fileStatus.getLegalBasisVersion(),
|
componentMappingVersions));
|
||||||
dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L),
|
|
||||||
Math.max(fileStatus.getDossierDictionaryVersion(), 0),
|
|
||||||
dossierDictionaryVersion);
|
|
||||||
}
|
}
|
||||||
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired);
|
return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private String buildMessage(FileModel fileStatus,
|
||||||
|
boolean rulesVersionMatches,
|
||||||
|
Map<VersionType, Long> dossierTemplateVersions,
|
||||||
|
boolean componentRulesVersionMatches,
|
||||||
|
boolean dictionaryVersionMatches,
|
||||||
|
boolean legalBasisVersionMatches,
|
||||||
|
boolean dossierDictionaryVersionMatches,
|
||||||
|
Long dossierDictionaryVersion,
|
||||||
|
boolean mappingVersionAllMatch,
|
||||||
|
Map<String, Integer> componentMappingVersions) {
|
||||||
|
|
||||||
|
StringBuilder messageBuilder = new StringBuilder();
|
||||||
|
messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> ");
|
||||||
|
|
||||||
|
boolean needComma = false;
|
||||||
|
|
||||||
|
if (rulesVersionMatches) {
|
||||||
|
messageBuilder.append("ruleVersions: ").append(fileStatus.getRulesVersion()).append("/").append(dossierTemplateVersions.getOrDefault(RULES, -1L));
|
||||||
|
needComma = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (componentRulesVersionMatches) {
|
||||||
|
if (needComma) {
|
||||||
|
messageBuilder.append(", ");
|
||||||
|
}
|
||||||
|
messageBuilder.append("componentRuleVersions: ")
|
||||||
|
.append(fileStatus.getComponentRulesVersion())
|
||||||
|
.append("/")
|
||||||
|
.append(dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L));
|
||||||
|
needComma = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dictionaryVersionMatches) {
|
||||||
|
if (needComma) {
|
||||||
|
messageBuilder.append(", ");
|
||||||
|
}
|
||||||
|
messageBuilder.append("dictionaryVersions: ").append(fileStatus.getDictionaryVersion()).append("/").append(dossierTemplateVersions.getOrDefault(DICTIONARY, -1L));
|
||||||
|
needComma = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (legalBasisVersionMatches) {
|
||||||
|
if (needComma) {
|
||||||
|
messageBuilder.append(", ");
|
||||||
|
}
|
||||||
|
messageBuilder.append("legalBasisVersions: ").append(fileStatus.getLegalBasisVersion()).append("/").append(dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L));
|
||||||
|
needComma = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dossierDictionaryVersionMatches) {
|
||||||
|
if (needComma) {
|
||||||
|
messageBuilder.append(", ");
|
||||||
|
}
|
||||||
|
messageBuilder.append("dossierDictionaryVersions: ").append(Math.max(fileStatus.getDossierDictionaryVersion(), 0)).append("/").append(dossierDictionaryVersion);
|
||||||
|
needComma = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mappingVersionAllMatch) {
|
||||||
|
if (needComma) {
|
||||||
|
messageBuilder.append(", ");
|
||||||
|
}
|
||||||
|
messageBuilder.append("componentMappingVersions: ").append(buildMappingVersionMatchesString(fileStatus.getComponentMappingVersions(), componentMappingVersions));
|
||||||
|
needComma = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return messageBuilder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private String buildMappingVersionMatchesString(Map<String, Integer> fileComponentMappingVersions, Map<String, Integer> dbComponentMappingVersions) {
|
||||||
|
|
||||||
|
Set<String> allMappingNames = new HashSet<>();
|
||||||
|
allMappingNames.addAll(fileComponentMappingVersions.keySet());
|
||||||
|
allMappingNames.addAll(dbComponentMappingVersions.keySet());
|
||||||
|
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
allMappingNames.stream()
|
||||||
|
.sorted()
|
||||||
|
.forEach(mappingName -> {
|
||||||
|
long fileVersion = fileComponentMappingVersions.getOrDefault(mappingName, -1);
|
||||||
|
long dbVersion = dbComponentMappingVersions.getOrDefault(mappingName, -1);
|
||||||
|
if (fileVersion != dbVersion) {
|
||||||
|
sb.append(mappingName).append(": ").append(fileVersion).append("/").append(dbVersion);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static boolean mappingVersionsEqual(FileModel fileStatus, Map<String, Integer> componentMappingVersions) {
|
||||||
|
|
||||||
|
return fileStatus.getComponentMappingVersions().keySet().equals(componentMappingVersions.keySet()) && fileStatus.getComponentMappingVersions().keySet()
|
||||||
|
.stream()
|
||||||
|
.allMatch(name -> fileStatus.getComponentMappingVersions()
|
||||||
|
.get(name).equals(componentMappingVersions.get(name)));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
private Map<VersionType, Long> buildVersionData(String dossierTemplateId) {
|
private Map<VersionType, Long> buildVersionData(String dossierTemplateId) {
|
||||||
|
|
||||||
var versions = new HashMap<VersionType, Long>();
|
var versions = new HashMap<VersionType, Long>();
|
||||||
|
|||||||
@ -69,7 +69,9 @@ public class AutomaticAnalysisJob implements Job {
|
|||||||
if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) {
|
if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) {
|
||||||
log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId());
|
log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId());
|
||||||
return;
|
return;
|
||||||
}var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
|
}
|
||||||
|
|
||||||
|
var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE);
|
||||||
if (redactionQueueInfo != null) {
|
if (redactionQueueInfo != null) {
|
||||||
log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
|
log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers",
|
||||||
tenant.getTenantId(),
|
tenant.getTenantId(),
|
||||||
|
|||||||
@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
|
|||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||||
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier;
|
import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier;
|
||||||
@ -23,6 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Websock
|
|||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||||
@ -121,11 +123,17 @@ public class FileStatusPersistenceService {
|
|||||||
long duration,
|
long duration,
|
||||||
long dossierDictionaryVersion,
|
long dossierDictionaryVersion,
|
||||||
int analysisVersion,
|
int analysisVersion,
|
||||||
int analysisNumber) {
|
int analysisNumber,
|
||||||
|
List<ComponentMappingMetadata> usedComponentMappings) {
|
||||||
|
|
||||||
if (isFileDeleted(fileId)) {
|
if (isFileDeleted(fileId)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
List<FileEntityComponentMappingVersionEntity> versionEntities = usedComponentMappings.stream()
|
||||||
|
.map(cm -> new FileEntityComponentMappingVersionEntity(cm.getName(), cm.getVersion()))
|
||||||
|
.toList();
|
||||||
|
|
||||||
fileRepository.updateProcessingStatus(fileId,
|
fileRepository.updateProcessingStatus(fileId,
|
||||||
numberOfPages,
|
numberOfPages,
|
||||||
ProcessingStatus.PROCESSED,
|
ProcessingStatus.PROCESSED,
|
||||||
@ -139,7 +147,8 @@ public class FileStatusPersistenceService {
|
|||||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
||||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS),
|
||||||
analysisNumber,
|
analysisNumber,
|
||||||
calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED));
|
calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED),
|
||||||
|
versionEntities);
|
||||||
|
|
||||||
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.FINISHED, analysisNumber);
|
websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.FINISHED, analysisNumber);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import org.springframework.data.jpa.repository.Query;
|
|||||||
import org.springframework.data.repository.query.Param;
|
import org.springframework.data.repository.query.Param;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||||
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection;
|
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection;
|
||||||
@ -47,12 +48,22 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
|||||||
|
|
||||||
|
|
||||||
@Modifying
|
@Modifying
|
||||||
@Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, "
|
@Query("""
|
||||||
+ "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, "
|
update FileEntity f set f.numberOfPages = :numberOfPages, \
|
||||||
+ "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, "
|
f.processingStatus = :processingStatus, \
|
||||||
+ "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, "
|
f.dictionaryVersion = :dictionaryVersion, \
|
||||||
+ "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter "
|
f.rulesVersion = :rulesVersion, \
|
||||||
+ "where f.id = :fileId")
|
f.componentRulesVersion = :componentRulesVersion, \
|
||||||
|
f.legalBasisVersion = :legalBasisVersion, \
|
||||||
|
f.analysisDuration = :analysisDuration, \
|
||||||
|
f.dossierDictionaryVersion = :dossierDictionaryVersion, \
|
||||||
|
f.analysisVersion = :analysisVersion, \
|
||||||
|
f.numberOfAnalyses = :analysisNumber, \
|
||||||
|
f.lastUpdated = :lastUpdated, \
|
||||||
|
f.lastProcessed = :lastProcessed, \
|
||||||
|
f.processingErrorCounter = :processingErrorCounter, \
|
||||||
|
f.componentMappingVersions = :componentMappingVersions \
|
||||||
|
where f.id = :fileId""")
|
||||||
void updateProcessingStatus(@Param("fileId") String fileId,
|
void updateProcessingStatus(@Param("fileId") String fileId,
|
||||||
@Param("numberOfPages") int numberOfPages,
|
@Param("numberOfPages") int numberOfPages,
|
||||||
@Param("processingStatus") ProcessingStatus processingStatus,
|
@Param("processingStatus") ProcessingStatus processingStatus,
|
||||||
@ -66,7 +77,9 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
|||||||
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
||||||
@Param("lastProcessed") OffsetDateTime lastProcessed,
|
@Param("lastProcessed") OffsetDateTime lastProcessed,
|
||||||
@Param("analysisNumber") int analysisNumber,
|
@Param("analysisNumber") int analysisNumber,
|
||||||
@Param("processingErrorCounter") int processingErrorCounter);
|
@Param("processingErrorCounter") int processingErrorCounter,
|
||||||
|
@Param("componentMappingVersions")
|
||||||
|
List<FileEntityComponentMappingVersionEntity> versionEntities);
|
||||||
|
|
||||||
|
|
||||||
@Modifying
|
@Modifying
|
||||||
@ -168,12 +181,13 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
|||||||
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
||||||
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
|
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
|
||||||
|
|
||||||
|
|
||||||
@Modifying
|
@Modifying
|
||||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id in (:fileIds)")
|
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id in (:fileIds)")
|
||||||
int softDeleteFiles(@Param("fileIds") List<String> fileIds,
|
int softDeleteFiles(@Param("fileIds") List<String> fileIds,
|
||||||
@Param("processingStatus") ProcessingStatus processingStatus,
|
@Param("processingStatus") ProcessingStatus processingStatus,
|
||||||
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
@Param("lastUpdated") OffsetDateTime lastUpdated,
|
||||||
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
|
@Param("softDeletedTime") OffsetDateTime softDeletedTime);
|
||||||
|
|
||||||
|
|
||||||
@Modifying
|
@Modifying
|
||||||
@ -380,10 +394,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
|||||||
+ " when f.deleted is not null then f.deleted "
|
+ " when f.deleted is not null then f.deleted "
|
||||||
+ "end "
|
+ "end "
|
||||||
+ "where f.id in (:fileIds)")
|
+ "where f.id in (:fileIds)")
|
||||||
int hardDeleteFiles(@Param("fileIds") List<String> fileIds,
|
int hardDeleteFiles(@Param("fileIds") List<String> fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime);
|
||||||
@Param("processingStatus") ProcessingStatus processingStatus,
|
|
||||||
@Param("deletionTime") OffsetDateTime deletionTime);
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,7 +1,9 @@
|
|||||||
package com.iqser.red.service.persistence.management.v1.processor.utils;
|
package com.iqser.red.service.persistence.management.v1.processor.utils;
|
||||||
|
|
||||||
import java.util.function.BiConsumer;
|
import java.util.function.BiConsumer;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity;
|
||||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||||
@ -14,6 +16,9 @@ public class FileModelMapper implements BiConsumer<FileEntity, FileModel> {
|
|||||||
fileEntity.getFileAttributes()
|
fileEntity.getFileAttributes()
|
||||||
.forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue()));
|
.forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue()));
|
||||||
fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp()));
|
fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp()));
|
||||||
|
fileModel.setComponentMappingVersions(fileEntity.getComponentMappingVersions()
|
||||||
|
.stream()
|
||||||
|
.collect(Collectors.toMap(FileEntityComponentMappingVersionEntity::getName, FileEntityComponentMappingVersionEntity::getVersion)));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -205,3 +205,5 @@ databaseChangeLog:
|
|||||||
file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml
|
file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml
|
||||||
- include:
|
- include:
|
||||||
file: db/changelog/tenant/127-add-component-mapping-table.yaml
|
file: db/changelog/tenant/127-add-component-mapping-table.yaml
|
||||||
|
- include:
|
||||||
|
file: db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml
|
||||||
|
|||||||
@ -0,0 +1,31 @@
|
|||||||
|
databaseChangeLog:
|
||||||
|
- changeSet:
|
||||||
|
id: add_component_mapping_version_to_file_entity
|
||||||
|
author: kilian
|
||||||
|
changes:
|
||||||
|
- createTable:
|
||||||
|
tableName: file_entity_component_mapping_versions
|
||||||
|
columns:
|
||||||
|
- column:
|
||||||
|
name: file_entity_id
|
||||||
|
type: VARCHAR(255)
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: name
|
||||||
|
type: VARCHAR(255)
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- column:
|
||||||
|
name: version
|
||||||
|
type: INT
|
||||||
|
constraints:
|
||||||
|
nullable: false
|
||||||
|
- addForeignKeyConstraint:
|
||||||
|
baseColumnNames: file_entity_id
|
||||||
|
baseTableName: file_entity_component_mapping_versions
|
||||||
|
constraintName: fk_component_mapping_version_entity_file
|
||||||
|
referencedColumnNames: id
|
||||||
|
referencedTableName: file
|
||||||
|
onDelete: CASCADE
|
||||||
|
onUpdate: CASCADE
|
||||||
@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
|||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.time.temporal.ChronoUnit;
|
import java.time.temporal.ChronoUnit;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@ -298,7 +299,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
|
|||||||
var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k);
|
var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k);
|
||||||
|
|
||||||
if (k == 1) {
|
if (k == 1) {
|
||||||
fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1);
|
fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1, Collections.emptyList());
|
||||||
reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k))));
|
reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k))));
|
||||||
}
|
}
|
||||||
if (k == 2) {
|
if (k == 2) {
|
||||||
|
|||||||
@ -1,8 +1,10 @@
|
|||||||
package com.iqser.red.service.persistence.service.v1.api.shared.model;
|
package com.iqser.red.service.persistence.service.v1.api.shared.model;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
@ -37,4 +39,6 @@ public class AnalyzeResult {
|
|||||||
|
|
||||||
private Set<FileAttribute> addedFileAttributes;
|
private Set<FileAttribute> addedFileAttributes;
|
||||||
|
|
||||||
|
private List<ComponentMappingMetadata> usedComponentMappings;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,38 @@
|
|||||||
|
package com.iqser.red.service.persistence.service.v1.api.shared.model.component;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import lombok.AccessLevel;
|
||||||
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.NonNull;
|
||||||
|
import lombok.experimental.FieldDefaults;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||||
|
public class ComponentMappingMetadata {
|
||||||
|
|
||||||
|
String id;
|
||||||
|
|
||||||
|
String name;
|
||||||
|
|
||||||
|
String fileName;
|
||||||
|
|
||||||
|
String storageId;
|
||||||
|
|
||||||
|
Integer version;
|
||||||
|
|
||||||
|
List<String> columnLabels;
|
||||||
|
|
||||||
|
Integer numberOfLines;
|
||||||
|
|
||||||
|
String encoding;
|
||||||
|
|
||||||
|
char delimiter;
|
||||||
|
|
||||||
|
}
|
||||||
@ -73,6 +73,7 @@ public class FileModel {
|
|||||||
private OffsetDateTime fileManipulationDate;
|
private OffsetDateTime fileManipulationDate;
|
||||||
private boolean hasHighlights;
|
private boolean hasHighlights;
|
||||||
private FileErrorInfo fileErrorInfo;
|
private FileErrorInfo fileErrorInfo;
|
||||||
|
private Map<String, Integer> componentMappingVersions = new HashMap<>();
|
||||||
|
|
||||||
|
|
||||||
public long getFileSize() {
|
public long getFileSize() {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user