DM-285: update component rules version with AnalyzeResult

This commit is contained in:
Kilian Schuettler 2023-09-13 14:24:25 +02:00
parent 9bca6b406a
commit 31890e6434
9 changed files with 42 additions and 16 deletions

View File

@ -435,6 +435,7 @@ public class FileStatusService {
analyzeResult.getNumberOfPages(), analyzeResult.getNumberOfPages(),
analyzeResult.getDictionaryVersion(), analyzeResult.getDictionaryVersion(),
analyzeResult.getRulesVersion(), analyzeResult.getRulesVersion(),
analyzeResult.getComponentRulesVersion(),
analyzeResult.getLegalBasisVersion(), analyzeResult.getLegalBasisVersion(),
analyzeResult.getDuration(), analyzeResult.getDuration(),
analyzeResult.getDossierDictionaryVersion(), analyzeResult.getDossierDictionaryVersion(),

View File

@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException; import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
@ -152,8 +153,8 @@ public class ReanalysisRequiredStatusService {
var versions = new HashMap<VersionType, Long>(); var versions = new HashMap<VersionType, Long>();
versions.put(RULES, rulesPersistenceService.getRules(dossierTemplateId, RuleFileType.ENTITY).getVersion()); versions.put(RULES, getRulesVersion(dossierTemplateId, RuleFileType.ENTITY));
versions.put(COMPONENT_RULES, rulesPersistenceService.getRules(dossierTemplateId, RuleFileType.COMPONENT).getVersion()); versions.put(COMPONENT_RULES, getRulesVersion(dossierTemplateId, RuleFileType.COMPONENT));
versions.put(DICTIONARY, dictionaryPersistenceService.getVersion(dossierTemplateId)); versions.put(DICTIONARY, dictionaryPersistenceService.getVersion(dossierTemplateId));
versions.put(LEGAL_BASIS, legalBasisMappingPersistenceService.getVersion(dossierTemplateId)); versions.put(LEGAL_BASIS, legalBasisMappingPersistenceService.getVersion(dossierTemplateId));
@ -161,6 +162,17 @@ public class ReanalysisRequiredStatusService {
} }
private Long getRulesVersion(String dossierTemplateId, RuleFileType ruleFileType) {
try {
return rulesPersistenceService.getRules(dossierTemplateId, ruleFileType).getVersion();
} catch (NotFoundException e) {
log.info(e.getMessage());
return -1L;
}
}
private Long getDossierVersionData(String dossierId) { private Long getDossierVersionData(String dossierId) {
return dictionaryPersistenceService.getVersionForDossier(dossierId); return dictionaryPersistenceService.getVersionForDossier(dossierId);

View File

@ -373,7 +373,7 @@ public class ManualRedactionService {
if (manualRedactions.getLegalBasisChanges() != null) { if (manualRedactions.getLegalBasisChanges() != null) {
manualRedactions.getLegalBasisChanges().forEach(e -> { manualRedactions.getLegalBasisChanges().forEach(e -> {
if (!e.getStatus().equals(AnnotationStatus.REQUESTED) && e.getProcessedDate() == null) { if (!e.getStatus().equals(AnnotationStatus.REQUESTED) && e.getProcessedDate() == null) {
resizeRedactionPersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId()); legalBasisChangePersistenceService.markAsProcessed(e.getAnnotationId(), e.getFileId());
} }
}); });
} }

View File

@ -104,6 +104,7 @@ public class FileStatusPersistenceService {
int numberOfPages, int numberOfPages,
long dictionaryVersion, long dictionaryVersion,
long rulesVersion, long rulesVersion,
long componentRulesVersion,
long legalBasisVersion, long legalBasisVersion,
long duration, long duration,
long dossierDictionaryVersion, long dossierDictionaryVersion,
@ -118,6 +119,7 @@ public class FileStatusPersistenceService {
ProcessingStatus.PROCESSED, ProcessingStatus.PROCESSED,
dictionaryVersion, dictionaryVersion,
rulesVersion, rulesVersion,
componentRulesVersion,
legalBasisVersion, legalBasisVersion,
duration, duration,
dossierDictionaryVersion, dossierDictionaryVersion,

View File

@ -84,4 +84,10 @@ public class LegalBasisChangePersistenceService {
} }
public void markAsProcessed(String annotationId, String fileId) {
legalBasisChangeRepository.markAsProcessed(new AnnotationEntityId(annotationId, fileId), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
} }

View File

@ -3,8 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.List; import java.util.List;
import jakarta.transaction.Transactional;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.Query;
@ -16,6 +14,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import jakarta.transaction.Transactional;
public interface FileRepository extends JpaRepository<FileEntity, String> { public interface FileRepository extends JpaRepository<FileEntity, String> {
boolean existsByDossierIdAndLastUpdatedIsAfter(String dossierId, OffsetDateTime since); boolean existsByDossierIdAndLastUpdatedIsAfter(String dossierId, OffsetDateTime since);
@ -40,12 +40,13 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying @Modifying
@Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.componentRulesVersion = :componentRulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
void updateProcessingStatus(String fileId, void updateProcessingStatus(String fileId,
int numberOfPages, int numberOfPages,
ProcessingStatus processingStatus, ProcessingStatus processingStatus,
long dictionaryVersion, long dictionaryVersion,
long rulesVersion, long rulesVersion,
long componentRulesVersion,
long legalBasisVersion, long legalBasisVersion,
long analysisDuration, long analysisDuration,
long dossierDictionaryVersion, long dossierDictionaryVersion,
@ -80,6 +81,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId")
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter); void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter);
@Modifying @Modifying
@Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service, f.errorTimestamp = :timestamp where f.id = :fileId") @Query("update FileEntity f set f.errorCause = :cause, f.errorQueue = :queue, f.errorService = :service, f.errorTimestamp = :timestamp where f.id = :fileId")
void updateStatusErrorInfo(String fileId, String cause, String queue, String service, OffsetDateTime timestamp); void updateStatusErrorInfo(String fileId, String cause, String queue, String service, OffsetDateTime timestamp);
@ -183,10 +185,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated); void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated);
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )" + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries")
+ " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'ERROR' )"
+ " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null "
+ " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries")
List<FileEntity> getAllRelevantStatusesForReanalysisScheduler(int maxRetries); List<FileEntity> getAllRelevantStatusesForReanalysisScheduler(int maxRetries);

View File

@ -31,4 +31,9 @@ public interface LegalBasisChangeRepository extends JpaRepository<ManualLegalBas
@Query("select mlbc from ManualLegalBasisChangeEntity mlbc where mlbc.id.fileId = :fileId and (:includeDeletions = true or mlbc.softDeletedTime is null)") @Query("select mlbc from ManualLegalBasisChangeEntity mlbc where mlbc.id.fileId = :fileId and (:includeDeletions = true or mlbc.softDeletedTime is null)")
List<ManualLegalBasisChangeEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions); List<ManualLegalBasisChangeEntity> findByFileIdIncludeDeletions(String fileId, boolean includeDeletions);
@Modifying
@Query("update ManualLegalBasisChangeEntity mlbc set mlbc.processedDate = :processedDate where mlbc.id = :annotationEntityId")
void markAsProcessed(AnnotationEntityId annotationEntityId, OffsetDateTime processedDate);
} }

View File

@ -286,20 +286,20 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
for (int k = 0; k < 6; k++) { for (int k = 0; k < 6; k++) {
var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k); var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k);
if (k == 1){ if (k == 1) {
fileStatusPersistenceService.updateProcessingStatus(fileId, k, 0L, 0L, 0L, 0L, 0L, 1, 1); fileStatusPersistenceService.updateProcessingStatus(fileId, k, 0L, 0L, 0L, 0L, 0L, 0L, 1, 1);
reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k)))); reanalysisClient.excludePages(dossier.getId(), fileId, new PageExclusionRequest(List.of(new PageRange(k, k))));
} }
if (k ==2) { if (k == 2) {
fileManagementClient.deleteFile(dossier.getId(), fileId); fileManagementClient.deleteFile(dossier.getId(), fileId);
} }
if (k == 3){ if (k == 3) {
fileManagementClient.hardDeleteFiles(dossier.getId(), Set.of(fileId)); fileManagementClient.hardDeleteFiles(dossier.getId(), Set.of(fileId));
} }
if (k == 4){ if (k == 4) {
fileClient.setStatusUnderReview(dossier.getId(), fileId, userId); fileClient.setStatusUnderReview(dossier.getId(), fileId, userId);
} }
if (k == 5){ if (k == 5) {
fileClient.setStatusUnderApproval(dossier.getId(), fileId, userId); fileClient.setStatusUnderApproval(dossier.getId(), fileId, userId);
} }
} }

View File

@ -25,6 +25,7 @@ public class AnalyzeResult {
private long dictionaryVersion; private long dictionaryVersion;
private long dossierDictionaryVersion; private long dossierDictionaryVersion;
private long rulesVersion; private long rulesVersion;
private long componentRulesVersion;
private long legalBasisVersion; private long legalBasisVersion;
private boolean wasReanalyzed; private boolean wasReanalyzed;