From 844466812b30a3487cc47ee88a05e8a720a28b2a Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 24 May 2024 14:58:26 +0200 Subject: [PATCH] RED-8670: integrate table inference from research * introduce controllerAdvice to ControllerV2 package --- .../processor/entity/dossier/FileEntity.java | 5 + ...leEntityComponentMappingVersionEntity.java | 17 ++ .../mapper/ComponentMappingEntityMapper.java | 2 +- .../processor/service/FileStatusService.java | 4 +- .../ReanalysisRequiredStatusService.java | 152 +++++++++++++++--- .../service/job/AutomaticAnalysisJob.java | 4 +- .../FileStatusPersistenceService.java | 13 +- .../repository/FileRepository.java | 39 +++-- .../v1/processor/utils/FileModelMapper.java | 5 + .../db/changelog/db.changelog-tenant.yaml | 2 + ...dd-component-mapping-versions-to-file.yaml | 31 ++++ .../tests/DossierTemplateStatsTest.java | 3 +- .../v1/api/shared/model/AnalyzeResult.java | 4 + .../component/ComponentMappingMetadata.java | 38 +++++ .../dossier/file/FileModel.java | 1 + 15 files changed, 279 insertions(+), 41 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 61fda011a..f55773cd6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -14,8 +14,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; import jakarta.persistence.CascadeType; +import jakarta.persistence.CollectionTable; import jakarta.persistence.Column; import jakarta.persistence.Convert; +import jakarta.persistence.ElementCollection; import jakarta.persistence.Entity; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; @@ -197,6 +199,9 @@ public class FileEntity { @Column private OffsetDateTime errorTimestamp; + @ElementCollection(fetch = FetchType.EAGER) + private List componentMappingVersions; + public OffsetDateTime getLastOCRTime() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java new file mode 100644 index 000000000..f50700668 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntityComponentMappingVersionEntity.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.persistence.management.v1.processor.entity.dossier; + +import jakarta.persistence.Embeddable; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Embeddable +@AllArgsConstructor +@NoArgsConstructor +public class FileEntityComponentMappingVersionEntity { + + private String name; + private Integer version; + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java index 76f0df0f4..7fa8ff3be 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/mapper/ComponentMappingEntityMapper.java @@ -17,7 +17,7 @@ public interface ComponentMappingEntityMapper { ComponentMappingMetadata toComponentMappingMetaData(ComponentMappingEntity componentMappingMetaData); - List toComponentMappingMetaDataList(List componentMappingMetaData); + List toComponentMappingMetaDataList(List componentMappingEntities); ComponentMappingEntity toComponentMappingEntity(ComponentMappingMetadata componentMappingSummary); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index ee262b7a1..0440de55f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.NotFo import com.iqser.red.service.persistence.management.v1.processor.model.websocket.AnalyseStatus; import com.iqser.red.service.persistence.management.v1.processor.model.websocket.FileEventType; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; + import org.apache.commons.lang3.StringUtils; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; @@ -520,7 +521,8 @@ public class FileStatusService { analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), - analyzeResult.getAnalysisNumber()); + analyzeResult.getAnalysisNumber(), + analyzeResult.getUsedComponentMappings()); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java index 453f1e7ea..234b3cee3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisRequiredStatusService.java @@ -7,8 +7,11 @@ import static com.iqser.red.service.persistence.management.v1.processor.service. import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -19,23 +22,28 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class ReanalysisRequiredStatusService { - private final DictionaryPersistenceService dictionaryPersistenceService; - private final RulesPersistenceService rulesPersistenceService; - private final DossierPersistenceService dossierPersistenceService; - private final LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + DictionaryPersistenceService dictionaryPersistenceService; + RulesPersistenceService rulesPersistenceService; + DossierPersistenceService dossierPersistenceService; + LegalBasisMappingPersistenceService legalBasisMappingPersistenceService; + ComponentMappingService componentMappingService; public FileModel enhanceFileStatusWithAnalysisRequirements(FileModel fileModel) { @@ -118,39 +126,141 @@ public class ReanalysisRequiredStatusService { Map dossierVersionMap) { // get relevant versions - var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), k -> buildVersionData(dossier.getDossierTemplateId())); - var dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); + Map dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), + k -> buildVersionData(dossier.getDossierTemplateId())); + + Map componentMappingVersions = componentMappingService.getMetaDataByDossierTemplateId(dossier.getDossierTemplateId()) + .stream() + .collect(Collectors.toMap(ComponentMappingMetadata::getName, ComponentMappingMetadata::getVersion)); + + Long dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); // compute matches + var mappingVersionAllMatch = mappingVersionsEqual(fileStatus, componentMappingVersions); var rulesVersionMatches = fileStatus.getRulesVersion() == dossierTemplateVersions.getOrDefault(RULES, -1L); var componentRulesVersionMatches = fileStatus.getComponentRulesVersion() == dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L); var dictionaryVersionMatches = fileStatus.getDictionaryVersion() == dossierTemplateVersions.getOrDefault(DICTIONARY, -1L); var legalBasisVersionMatches = fileStatus.getLegalBasisVersion() == dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L); var dossierDictionaryVersionMatches = Math.max(fileStatus.getDossierDictionaryVersion(), 0) == dossierDictionaryVersion; - var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches; + var reanalysisRequired = !dictionaryVersionMatches || !dossierDictionaryVersionMatches || !mappingVersionAllMatch; var fullAnalysisRequired = !rulesVersionMatches || !componentRulesVersionMatches || !legalBasisVersionMatches; + if (reanalysisRequired || fullAnalysisRequired) { - log.info( - "For file: {}-{} analysis is required because -> ruleVersionMatches: {}/{}, componentRuleVersionMatches {}/{}, dictionaryVersionMatches: {}/{}, legalBasisVersionMatches: {}/{}, dossierDictionaryVersionMatches: {}/{}", - fileStatus.getId(), - fileStatus.getFilename(), - fileStatus.getRulesVersion(), - dossierTemplateVersions.getOrDefault(RULES, -1L), - fileStatus.getComponentRulesVersion(), - dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L), - fileStatus.getDictionaryVersion(), - dossierTemplateVersions.getOrDefault(DICTIONARY, -1L), - fileStatus.getLegalBasisVersion(), - dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L), - Math.max(fileStatus.getDossierDictionaryVersion(), 0), - dossierDictionaryVersion); + + log.info(buildMessage(fileStatus, + rulesVersionMatches, + dossierTemplateVersions, + componentRulesVersionMatches, + dictionaryVersionMatches, + legalBasisVersionMatches, + dossierDictionaryVersionMatches, + dossierDictionaryVersion, + mappingVersionAllMatch, + componentMappingVersions)); } return new AnalysisRequiredResult(reanalysisRequired, fullAnalysisRequired); } + private String buildMessage(FileModel fileStatus, + boolean rulesVersionMatches, + Map dossierTemplateVersions, + boolean componentRulesVersionMatches, + boolean dictionaryVersionMatches, + boolean legalBasisVersionMatches, + boolean dossierDictionaryVersionMatches, + Long dossierDictionaryVersion, + boolean mappingVersionAllMatch, + Map componentMappingVersions) { + + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append("For file: ").append(fileStatus.getId()).append("-").append(fileStatus.getFilename()).append(" analysis is required because -> "); + + boolean needComma = false; + + if (rulesVersionMatches) { + messageBuilder.append("ruleVersions: ").append(fileStatus.getRulesVersion()).append("/").append(dossierTemplateVersions.getOrDefault(RULES, -1L)); + needComma = true; + } + + if (componentRulesVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("componentRuleVersions: ") + .append(fileStatus.getComponentRulesVersion()) + .append("/") + .append(dossierTemplateVersions.getOrDefault(COMPONENT_RULES, -1L)); + needComma = true; + } + + if (dictionaryVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("dictionaryVersions: ").append(fileStatus.getDictionaryVersion()).append("/").append(dossierTemplateVersions.getOrDefault(DICTIONARY, -1L)); + needComma = true; + } + + if (legalBasisVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("legalBasisVersions: ").append(fileStatus.getLegalBasisVersion()).append("/").append(dossierTemplateVersions.getOrDefault(LEGAL_BASIS, -1L)); + needComma = true; + } + + if (dossierDictionaryVersionMatches) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("dossierDictionaryVersions: ").append(Math.max(fileStatus.getDossierDictionaryVersion(), 0)).append("/").append(dossierDictionaryVersion); + needComma = true; + } + + if (mappingVersionAllMatch) { + if (needComma) { + messageBuilder.append(", "); + } + messageBuilder.append("componentMappingVersions: ").append(buildMappingVersionMatchesString(fileStatus.getComponentMappingVersions(), componentMappingVersions)); + needComma = true; + } + + return messageBuilder.toString(); + } + + + private String buildMappingVersionMatchesString(Map fileComponentMappingVersions, Map dbComponentMappingVersions) { + + Set allMappingNames = new HashSet<>(); + allMappingNames.addAll(fileComponentMappingVersions.keySet()); + allMappingNames.addAll(dbComponentMappingVersions.keySet()); + + StringBuilder sb = new StringBuilder(); + allMappingNames.stream() + .sorted() + .forEach(mappingName -> { + long fileVersion = fileComponentMappingVersions.getOrDefault(mappingName, -1); + long dbVersion = dbComponentMappingVersions.getOrDefault(mappingName, -1); + if (fileVersion != dbVersion) { + sb.append(mappingName).append(": ").append(fileVersion).append("/").append(dbVersion); + } + }); + return sb.toString(); + } + + + private static boolean mappingVersionsEqual(FileModel fileStatus, Map componentMappingVersions) { + + return fileStatus.getComponentMappingVersions().keySet().equals(componentMappingVersions.keySet()) && fileStatus.getComponentMappingVersions().keySet() + .stream() + .allMatch(name -> fileStatus.getComponentMappingVersions() + .get(name).equals(componentMappingVersions.get(name))); + } + + private Map buildVersionData(String dossierTemplateId) { var versions = new HashMap(); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java index ebc985b83..c4794dbf6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/AutomaticAnalysisJob.java @@ -69,7 +69,9 @@ public class AutomaticAnalysisJob implements Job { if (!saasMigrationStatusPersistenceService.migrationFinishedForTenant()) { log.info("[Tenant:{}] Skipping scheduling as there are files that require migration.", tenant.getTenantId()); return; - }var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); + } + + var redactionQueueInfo = amqpAdmin.getQueueInfo(MessagingConfiguration.REDACTION_QUEUE); if (redactionQueueInfo != null) { log.debug("[Tenant:{}] Checking queue status to see if background analysis can happen. Currently {} holds {} elements and has {} consumers", tenant.getTenantId(), diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index d47301caa..28c135809 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -14,6 +14,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.model.FileIdentifier; @@ -23,6 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.Websock import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -121,11 +123,17 @@ public class FileStatusPersistenceService { long duration, long dossierDictionaryVersion, int analysisVersion, - int analysisNumber) { + int analysisNumber, + List usedComponentMappings) { if (isFileDeleted(fileId)) { return; } + + List versionEntities = usedComponentMappings.stream() + .map(cm -> new FileEntityComponentMappingVersionEntity(cm.getName(), cm.getVersion())) + .toList(); + fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, @@ -139,7 +147,8 @@ public class FileStatusPersistenceService { OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber, - calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED)); + calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED), + versionEntities); websocketService.sendAnalysisEvent(dossierId, fileId, AnalyseStatus.FINISHED, analysisNumber); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 775564d99..afe318ea4 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -10,6 +10,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection; @@ -47,12 +48,22 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " - + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, " - + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " - + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " - + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " - + "where f.id = :fileId") + @Query(""" + update FileEntity f set f.numberOfPages = :numberOfPages, \ + f.processingStatus = :processingStatus, \ + f.dictionaryVersion = :dictionaryVersion, \ + f.rulesVersion = :rulesVersion, \ + f.componentRulesVersion = :componentRulesVersion, \ + f.legalBasisVersion = :legalBasisVersion, \ + f.analysisDuration = :analysisDuration, \ + f.dossierDictionaryVersion = :dossierDictionaryVersion, \ + f.analysisVersion = :analysisVersion, \ + f.numberOfAnalyses = :analysisNumber, \ + f.lastUpdated = :lastUpdated, \ + f.lastProcessed = :lastProcessed, \ + f.processingErrorCounter = :processingErrorCounter, \ + f.componentMappingVersions = :componentMappingVersions \ + where f.id = :fileId""") void updateProcessingStatus(@Param("fileId") String fileId, @Param("numberOfPages") int numberOfPages, @Param("processingStatus") ProcessingStatus processingStatus, @@ -66,7 +77,9 @@ public interface FileRepository extends JpaRepository { @Param("lastUpdated") OffsetDateTime lastUpdated, @Param("lastProcessed") OffsetDateTime lastProcessed, @Param("analysisNumber") int analysisNumber, - @Param("processingErrorCounter") int processingErrorCounter); + @Param("processingErrorCounter") int processingErrorCounter, + @Param("componentMappingVersions") + List versionEntities); @Modifying @@ -168,12 +181,13 @@ public interface FileRepository extends JpaRepository { @Param("lastUpdated") OffsetDateTime lastUpdated, @Param("softDeletedTime") OffsetDateTime softDeletedTime); + @Modifying @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id in (:fileIds)") int softDeleteFiles(@Param("fileIds") List fileIds, - @Param("processingStatus") ProcessingStatus processingStatus, - @Param("lastUpdated") OffsetDateTime lastUpdated, - @Param("softDeletedTime") OffsetDateTime softDeletedTime); + @Param("processingStatus") ProcessingStatus processingStatus, + @Param("lastUpdated") OffsetDateTime lastUpdated, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); @Modifying @@ -380,10 +394,7 @@ public interface FileRepository extends JpaRepository { + " when f.deleted is not null then f.deleted " + "end " + "where f.id in (:fileIds)") - int hardDeleteFiles(@Param("fileIds") List fileIds, - @Param("processingStatus") ProcessingStatus processingStatus, - @Param("deletionTime") OffsetDateTime deletionTime); - + int hardDeleteFiles(@Param("fileIds") List fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java index 8f6fbc97a..fb29fe9e0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/FileModelMapper.java @@ -1,7 +1,9 @@ package com.iqser.red.service.persistence.management.v1.processor.utils; import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntityComponentMappingVersionEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; @@ -14,6 +16,9 @@ public class FileModelMapper implements BiConsumer { fileEntity.getFileAttributes() .forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue())); fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp())); + fileModel.setComponentMappingVersions(fileEntity.getComponentMappingVersions() + .stream() + .collect(Collectors.toMap(FileEntityComponentMappingVersionEntity::getName, FileEntityComponentMappingVersionEntity::getVersion))); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index 9cf34c503..470f13643 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -205,3 +205,5 @@ databaseChangeLog: file: db/changelog/tenant/126-add-experimental-flag-to-entity.yaml - include: file: db/changelog/tenant/127-add-component-mapping-table.yaml + - include: + file: db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml new file mode 100644 index 000000000..e9dbcefa2 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/128-add-component-mapping-versions-to-file.yaml @@ -0,0 +1,31 @@ +databaseChangeLog: + - changeSet: + id: add_component_mapping_version_to_file_entity + author: kilian + changes: + - createTable: + tableName: file_entity_component_mapping_versions + columns: + - column: + name: file_entity_id + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: name + type: VARCHAR(255) + constraints: + nullable: false + - column: + name: version + type: INT + constraints: + nullable: false + - addForeignKeyConstraint: + baseColumnNames: file_entity_id + baseTableName: file_entity_component_mapping_versions + constraintName: fk_component_mapping_version_entity_file + referencedColumnNames: id + referencedTableName: file + onDelete: CASCADE + onUpdate: CASCADE diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java index 55535290d..110b05765 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java @@ -5,6 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -298,7 +299,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k); if (k == 1) { - fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1); + fileStatusPersistenceService.updateProcessingStatus(dossier.getId(), fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1, Collections.emptyList()); reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k)))); } if (k == 2) { diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java index 5ef8e7b29..f6148ec5b 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/AnalyzeResult.java @@ -1,8 +1,10 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; +import java.util.List; import java.util.Set; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import lombok.AllArgsConstructor; import lombok.Builder; @@ -37,4 +39,6 @@ public class AnalyzeResult { private Set addedFileAttributes; + private List usedComponentMappings; + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java new file mode 100644 index 000000000..410a74afd --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/component/ComponentMappingMetadata.java @@ -0,0 +1,38 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model.component; + +import java.util.List; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ComponentMappingMetadata { + + String id; + + String name; + + String fileName; + + String storageId; + + Integer version; + + List columnLabels; + + Integer numberOfLines; + + String encoding; + + char delimiter; + +} diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java index 06e45cee0..6794ba1aa 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/dossier/file/FileModel.java @@ -73,6 +73,7 @@ public class FileModel { private OffsetDateTime fileManipulationDate; private boolean hasHighlights; private FileErrorInfo fileErrorInfo; + private Map componentMappingVersions = new HashMap<>(); public long getFileSize() {