diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java index 4102fd79f..9eb35e6a9 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/FileStatusProcessingUpdateResource.java @@ -33,8 +33,7 @@ public interface FileStatusProcessingUpdateResource { @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = STATUS_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE + "/analysis-successful", consumes = MediaType.APPLICATION_JSON_VALUE) - void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, - @RequestBody AnalyzeResult analyzeResult); + void analysisSuccessful(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody AnalyzeResult analyzeResult); @ResponseStatus(value = HttpStatus.OK) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java index bc525e583..92ef04941 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java @@ -1,13 +1,18 @@ package com.iqser.red.service.persistence.service.v1.api.resources; +import java.util.Set; + +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; + import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.*; - -import java.util.Set; @ResponseStatus(value = HttpStatus.NO_CONTENT) public interface ReanalysisResource { @@ -55,9 +60,11 @@ public interface ReanalysisResource { @PostMapping(value = IMPORT_REDACTIONS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE) void importRedactions(@RequestBody ByteContentDocument documentRequest); + @PostMapping(value = IMPORT_REDACTIONS_PATH + "/delete", consumes = MediaType.APPLICATION_JSON_VALUE) void deleteImportedRedactions(@RequestBody DeleteImportedRedactionsRequest deleteImportedRedactionsRequest); + @PostMapping(value = CONVERT_TEXT_HIGHLIGHTS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE) void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 00ecffe34..bc7821d09 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -169,6 +169,9 @@ public class FileEntity { @Column(name = "dossier_id") private String dossierId; + @Column + private Integer processingErrorCounter; + public boolean isSoftDeleted() { return deleted != null; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index b6425f981..341e71699 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -52,6 +52,7 @@ public class FileStatusPersistenceService { file.setLastUploaded(now); file.setLastUpdated(now); file.setFileManipulationDate(now); + file.setProcessingErrorCounter(0); fileRepository.save(file); } @@ -62,8 +63,8 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, - OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize); + fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize, calculateProcessingErrorCounter(fileId, ProcessingStatus.PRE_PROCESSED)); } @@ -73,27 +74,26 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSING_FAILED, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PRE_PROCESSING_FAILED)); } @Transactional - public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion, - long legalBasisVersion, long duration, long dossierDictionaryVersion, - int analysisVersion, int analysisNumber) { + public void updateProcessingStatus(String fileId, int numberOfPages, long dictionaryVersion, long rulesVersion, long legalBasisVersion, long duration, + long dossierDictionaryVersion, int analysisVersion, int analysisNumber) { if (isFileDeleted(fileId)) { return; } - fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, - legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber); + fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), analysisNumber, calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED)); } @Transactional - public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, - boolean hasSuggestions, boolean hasComments, boolean hasUpdates) { + public void updateFlags(String fileId, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates) { if (isFileDeleted(fileId)) { return; @@ -102,12 +102,6 @@ public class FileStatusPersistenceService { } - private boolean isFileDeleted(String fileId) { - - return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false); - } - - @Transactional public void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, boolean approval) { @@ -115,12 +109,10 @@ public class FileStatusPersistenceService { return; } if (approval) { - fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now() + fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now() .truncatedTo(ChronoUnit.MILLIS), true); } else { - fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), null); + fileRepository.updateWorkflowStatus(fileId, workflowStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null); } } @@ -134,11 +126,11 @@ public class FileStatusPersistenceService { } if (processingStatus == ProcessingStatus.PROCESSED) { // In case the file is updated to "processed", "lastProcessed" date should be updated to "now" - fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED)); } else { fileRepository.updateProcessingStatus(fileId, processingStatus, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS)); + .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, processingStatus)); } } @@ -170,8 +162,8 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + fileRepository.setUpdateStatusIndexingSuccessful(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), calculateProcessingErrorCounter(fileId, ProcessingStatus.PROCESSED)); } @@ -236,12 +228,10 @@ public class FileStatusPersistenceService { public List getActiveFiles(String dossierId) { - return fileRepository.findByDossierId(dossierId) - .stream() - .filter(f -> !f.isSoftOrHardDeleted()) - .collect(Collectors.toList()); + return fileRepository.findByDossierId(dossierId).stream().filter(f -> !f.isSoftOrHardDeleted()).collect(Collectors.toList()); } + public List getSoftDeletedFiles(List dossierIds) { return fileRepository.getSoftDeletedFiles(dossierIds); @@ -264,8 +254,7 @@ public class FileStatusPersistenceService { @Transactional public void softDelete(String fileId, OffsetDateTime softDeletedTime) { - int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), softDeletedTime); + int countUpdate = fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -282,8 +271,7 @@ public class FileStatusPersistenceService { }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); - fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), null); + fileRepository.setSoftDelete(fileId, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), null); } @@ -293,8 +281,7 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS)); + int updateCount = fileRepository.setAssignee(fileId, currentAssignee, lastReviewer, lastApprover, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (updateCount == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -307,8 +294,7 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS)); + int countUpdate = fileRepository.toggleExclusion(fileId, excluded, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -321,8 +307,7 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS)); + int countUpdate = fileRepository.toggleAutomaticAnalysis(fileId, excludedFromAutomaticAnalysis, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (countUpdate == 0) { throw new NotFoundException("Unknown file=" + fileId); } @@ -332,18 +317,17 @@ public class FileStatusPersistenceService { @Transactional public void overwriteFile(String fileId, String uploader, String filename, boolean keepManualRedactions) { - int countUpdate = 0; + int countUpdate; - if(keepManualRedactions) { + if (keepManualRedactions) { countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, OffsetDateTime.now() .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } else { - fileRepository.findById(fileId).ifPresent((file) -> { - file.setExcludedPages(new HashSet<>()); - }); + fileRepository.findById(fileId).ifPresent((file) -> file.setExcludedPages(new HashSet<>())); - countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.UNPROCESSED, WorkflowStatus.NEW, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } deleteFileAttributes(fileId); @@ -354,33 +338,13 @@ public class FileStatusPersistenceService { } - private List convertFileAttributes(String dossierId, FileEntity file, - Map fileAttributesMap) { - - var dossier = dossierService.getAndValidateDossier(dossierId); - List configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId()); - - return fileAttributesMap.entrySet().stream().map(entry -> { - var fa = new FileAttributeEntity(); - var id = new FileAttributeEntity.FileAttributeEntityId(); - id.setFileId(file.getId()); - id.setFileAttributeConfigId(configuration.stream() - .filter(c -> c.getId().equals(entry.getKey())) - .findAny().map(FileAttributeConfigEntity::getId) - .orElseThrow(() -> new BadRequestException("Invalid File Attribute Id"))); - fa.setFileAttributeId(id); - fa.setValue(StringUtils.isBlank(entry.getValue()) ? null : entry.getValue()); - return fa; - }).collect(Collectors.toList()); - } - - @Transactional public void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime changeDate) { fileRepository.setLastRedactionModificationDateForFile(fileId, changeDate, OffsetDateTime.now()); } + @Transactional public void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate) { @@ -399,6 +363,7 @@ public class FileStatusPersistenceService { return fileRepository.countSoftDeletedFilesPerDossierTemplateId(dossierTemplateId); } + public int countSoftDeletedFilesPerDossierId(String dossierId) { return fileRepository.countSoftDeletedFilesPerDossierId(dossierId); @@ -424,4 +389,48 @@ public class FileStatusPersistenceService { fileRepository.updateHasHighlights(fileId, hasHighlights); } + + private int calculateProcessingErrorCounter(String fileId, ProcessingStatus processingStatus) { + + switch (processingStatus) { + case ERROR: + case PRE_PROCESSING_FAILED: + return fileRepository.findById(fileId).map(FileEntity::getProcessingErrorCounter).orElse(0) + 1; + + case PROCESSED: + return 0; + + default: + return fileRepository.findById(fileId).map(FileEntity::getProcessingErrorCounter).orElse(0); + } + + } + + + private boolean isFileDeleted(String fileId) { + + return fileRepository.findById(fileId).map(FileEntity::isSoftOrHardDeleted).orElse(false); + } + + + private List convertFileAttributes(String dossierId, FileEntity file, Map fileAttributesMap) { + + var dossier = dossierService.getAndValidateDossier(dossierId); + List configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId()); + + return fileAttributesMap.entrySet().stream().map(entry -> { + var fa = new FileAttributeEntity(); + var id = new FileAttributeEntity.FileAttributeEntityId(); + id.setFileId(file.getId()); + id.setFileAttributeConfigId(configuration.stream() + .filter(c -> c.getId().equals(entry.getKey())) + .findAny() + .map(FileAttributeConfigEntity::getId) + .orElseThrow(() -> new BadRequestException("Invalid File Attribute Id"))); + fa.setFileAttributeId(id); + fa.setValue(StringUtils.isBlank(entry.getValue()) ? null : entry.getValue()); + return fa; + }).collect(Collectors.toList()); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 577f2d8e3..fdac4e25e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -1,88 +1,73 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; +import java.time.OffsetDateTime; +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; - -import java.time.OffsetDateTime; -import java.util.List; - public interface FileRepository extends JpaRepository { boolean existsByDossierIdAndLastUpdatedIsAfter(String dossierId, OffsetDateTime since); + List findByDossierId(String dossierId); + @Modifying - @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + - " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + - " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + - " e.lastUpdated = :lastUpdated " + - " where e.id =:fileId") - void updateFlags(String fileId, - OffsetDateTime lastUpdated, - boolean hasRedactions, - boolean hasHints, - boolean hasImages, - boolean hasSuggestions, - boolean hasComments, + @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId") + void updateFlags(String fileId, OffsetDateTime lastUpdated, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates); - @Modifying - @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + - "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + - "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + - "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + - "f.lastProcessed = :lastProcessed " + - "where f.id = :fileId") - void updateProcessingStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion, - long rulesVersion, long legalBasisVersion, long analysisDuration, long dossierDictionaryVersion, - int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber); @Modifying - @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + - " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId") + @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + void updateProcessingStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion, long rulesVersion, long legalBasisVersion, + long analysisDuration, long dossierDictionaryVersion, int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, + int analysisNumber, int processingErrorCounter); + + + @Modifying + @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId") void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate, boolean excludedFromAutomaticAnalysis); + @Modifying - @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + - " where f.id = :fileId") + @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + " where f.id = :fileId") void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate); - @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + - " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize " + - " where f.id = :fileId") - void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize); - - @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.fileSize = :fileSize where f.id = :fileId") void updateFileSize(String fileId, long fileSize); @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated " + - "where f.id = :fileId") - void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated); + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize, f.processingErrorCounter = :processingErrorCounter " + " where f.id = :fileId") + void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize, int processingErrorCounter); + @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed " + - "where f.id = :fileId") - void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed); + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter); + @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + - "f.lastIndexed = :lastIndexed where f.id = :fileId") - void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed); + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int processingErrorCounter); + + + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId") + void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed, int processingErrorCounter); @Modifying @@ -107,117 +92,91 @@ public interface FileRepository extends JpaRepository { @Modifying @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id = :fileId") - int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, - OffsetDateTime softDeletedTime); + int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime softDeletedTime); + @Modifying - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + - "f.hardDeletedTime = :hardDeletedTime, " + - "f.deleted = case " + - "when f.deleted is null then :deleted " + - "when f.deleted is not null then f.deleted " + - "end " + - "where f.id = :fileId") - int setHardDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, - OffsetDateTime hardDeletedTime, OffsetDateTime deleted); + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.hardDeletedTime = :hardDeletedTime, " + "f.deleted = case " + "when f.deleted is null then :deleted " + "when f.deleted is not null then f.deleted " + "end " + "where f.id = :fileId") + int setHardDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime hardDeletedTime, OffsetDateTime deleted); + @Modifying - @Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " + - "f.lastUpdated = :lastUpdated where f.id = :fileId") + @Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " + "f.lastUpdated = :lastUpdated where f.id = :fileId") int setAssignee(String fileId, String assignee, String lastReviewer, String lastApprover, OffsetDateTime lastUpdated); + @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.excluded = :excluded, f.lastUpdated = :lastUpdated where f.id = :fileId") int toggleExclusion(String fileId, boolean excluded, OffsetDateTime lastUpdated); + @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastUpdated = :lastUpdated where f.id = :fileId") int toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis, OffsetDateTime lastUpdated); @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " - + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " - + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " - + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " - + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " - + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " - + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false " + "where f.id = :fileId") + @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, f.processingErrorCounter = 0 where f.id = :fileId") int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated); - @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " - + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " - + "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " - + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " - + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " - + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " - + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false " + "where f.id = :fileId") - int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded, - OffsetDateTime lastUpdated); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + - " and ((f.deleted is not null and f.hardDeletedTime is null) or " + - " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0 where f.id = :fileId") + int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded, + OffsetDateTime lastUpdated); + + + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") int countSoftDeletedFilesPerDossierTemplateId(String dossierTemplateId); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" + - " and ((f.deleted is not null and f.hardDeletedTime is null) or " + - " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") + + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") int countSoftDeletedFilesPerDossierId(String dossierId); + @Query("select distinct f.dossierId from FileEntity f where f.lastUpdated > :since") List findDossierChangeByLastUpdatedIsAfter(OffsetDateTime since); + @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.redactionModificationDate = :redactionModificationDate, f.lastUpdated = :lastUpdated where f.id = :fileId") void setLastRedactionModificationDateForFile(String fileId, OffsetDateTime redactionModificationDate, OffsetDateTime lastUpdated); + @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.lastManualChangeDate = :lastManualChangeDate, f.lastUpdated = :lastUpdated where f.id = :fileId") void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated); - @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + - "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" + - "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + - "and f.deleted is null and f.hardDeletedTime is null") + + @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and f.deleted is null and f.hardDeletedTime is null") List getAllRelevantStatusesForReanalysisScheduler(); + @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.lastFileAttributeChange = :date, f.lastUpdated = :date where f.id = :fileId") void updateLastAttributeChangeDate(String fileId, OffsetDateTime date); + @Query("select f from FileEntity f where f.deleted is not null and f.hardDeletedTime is null and f.dossierId in :dossierIds") List getSoftDeletedFiles(List dossierIds); - @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + - "inner join DossierEntity d on d.id = f.dossierId " + - "where f.deleted is null and f.hardDeletedTime is null " + - "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + - "and d.dossierTemplateId = :dossierTemplateId " + - "group by f.processingStatus ") + + @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.processingStatus ") List countFilesByProcessingStatus(String dossierTemplateId); - @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + - "inner join DossierEntity d on d.id = f.dossierId " + - "where f.deleted is null and f.hardDeletedTime is null " + - "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + - "and d.dossierTemplateId = :dossierTemplateId " + - "group by f.workflowStatus ") + + @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.workflowStatus ") List countFilesByWorkflowStatus(String dossierTemplateId); - @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + - " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + - " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + - " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) + + @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) FilePageCountsProjection countPages(String dossierTemplateId); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + - "f.hardDeletedTime is null and f.deleted is null and " + - "d.dossierTemplateId = :dossierTemplateId and " + - "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") + + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + "f.hardDeletedTime is null and f.deleted is null and " + "d.dossierTemplateId = :dossierTemplateId and " + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") int countActiveFiles(String dossierTemplateId); + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index 8f2a87e5f..fcdbe921f 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -1,8 +1,21 @@ package com.iqser.red.service.peristence.v1.server.controller; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; -import com.iqser.red.service.pdftron.redaction.v1.api.model.*; +import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; @@ -15,24 +28,12 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.resources.ReanalysisResource; + import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; @Slf4j @RestController @@ -48,8 +49,7 @@ public class ReanalysisController implements ReanalysisResource { @Override - public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, - @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { + public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) { var relevantFiles = getAllFilesForDossier(dossierId, true); reanalyseFiles(dossierId, force, relevantFiles); @@ -94,8 +94,7 @@ public class ReanalysisController implements ReanalysisResource { fileStatusService.setStatusOcrProcessing(dossierId, fileId); } else { FileModel dossierFile = fileStatusService.getStatus(fileId); - if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus() - .equals(WorkflowStatus.APPROVED)) { + if (dossierFile.isSoftOrHardDeleted() || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { throw new ConflictException("Cannot analyse a deleted/approved file"); } if (dossierFile.getLastOCRTime() != null) { @@ -107,6 +106,7 @@ public class ReanalysisController implements ReanalysisResource { public void importRedactions(@RequestBody ByteContentDocument documentRequest) { + try { pDFTronRedactionClient.importRedactions(documentRequest); } catch (Exception e) { @@ -129,8 +129,7 @@ public class ReanalysisController implements ReanalysisResource { public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId, - @RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, - @RequestBody Set fileIds) { + @RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set fileIds) { indexingService.reindex(dossierId, fileIds, dropIndex); } @@ -141,8 +140,7 @@ public class ReanalysisController implements ReanalysisResource { boolean hasTextHighlights = pDFTronRedactionClient.convertTextHighlights(textHighlightRequest); fileStatusService.updateHasHighlights(textHighlightRequest.getFileId(), hasTextHighlights); - fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS)); + fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) { fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true, false); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 53798af8e..c3fc63e4e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -1,17 +1,17 @@ package com.iqser.red.service.peristence.v1.server.service; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.web.bind.annotation.RestController; + import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.search.v1.model.IndexMessageType; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; @Slf4j @RestController @@ -43,7 +43,7 @@ public class FileStatusProcessingUpdateService { fileStatusService.setStatusAnalyse(dossierId, fileId, false); //TODO This might be also priority depending on what was the pervious call. - fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null); + fileStatusService.addToAnalysisQueue(dossierId, fileId, false, null, false); } break; @@ -72,20 +72,18 @@ public class FileStatusProcessingUpdateService { } - public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse){ + public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse) { - fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(),untouchedDocumentResponse.getFileSize()); + fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(), untouchedDocumentResponse.getFileSize()); } - - public void preprocessingFailed(String dossierId, String fileId){ + public void preprocessingFailed(String dossierId, String fileId) { fileStatusService.updateProcessingStatusPreprocessingFailed(dossierId, fileId); } - public void ocrSuccessful(String dossierId, String fileId) { retryTemplate.execute(retryContext -> { @@ -111,16 +109,6 @@ public class FileStatusProcessingUpdateService { } - private void setStatusError(String dossierId, String fileId, String reason) { - - retryTemplate.execute(retryContext -> { - log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason); - fileStatusService.setStatusError(fileId); - return null; - }); - } - - public void indexing(String dossierId, String fileId) { retryTemplate.execute(retryContext -> { @@ -146,4 +134,14 @@ public class FileStatusProcessingUpdateService { setStatusError(dossierId, fileId, "indexingFailed"); } + + private void setStatusError(String dossierId, String fileId, String reason) { + + retryTemplate.execute(retryContext -> { + log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason); + fileStatusService.setStatusError(fileId); + return null; + }); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index a6b300dae..adc3b71b9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -1,5 +1,19 @@ package com.iqser.red.service.peristence.v1.server.service; +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.transaction.Transactional; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -11,7 +25,6 @@ import com.iqser.red.service.peristence.v1.server.model.Pdf2ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; @@ -20,7 +33,13 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -28,21 +47,10 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; + import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - -import javax.transaction.Transactional; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @Slf4j @Service @@ -75,10 +83,7 @@ public class FileStatusService { var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(); var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); - return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList) - .stream() - .filter(FileModel::isAnalysisRequired) - .collect(Collectors.toList()); + return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList()); } @@ -119,8 +124,8 @@ public class FileStatusService { public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) { - fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights,fileSize); - addToAnalysisQueue(dossierId, fileId, false, Set.of()); + fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize); + addToAnalysisQueue(dossierId, fileId, false, Set.of(), false); if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) { addToPdf2ImageQueue(dossierId, fileId); @@ -182,8 +187,7 @@ public class FileStatusService { @Transactional - public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse, - boolean triggeredManually) { + public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse, boolean triggeredManually) { log.info("Reprocessing file: {} from dossier {}", fileId, dossierId); @@ -199,8 +203,13 @@ public class FileStatusService { return; } + if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries() && !triggeredManually) { + log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter()); + return; + } + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.REPROCESS); - addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse); + addToAnalysisQueue(dossierId, fileId, priority, sectionsToReanalyse, triggeredManually); } @@ -217,8 +226,7 @@ public class FileStatusService { @Transactional - public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, - boolean requiresStructureAnalysis) { + public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -227,24 +235,25 @@ public class FileStatusService { return; } + if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) { + log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter()); + return; + } + if (requiresStructureAnalysis) { log.info("Delete text and NER entities from file {} in dossier {}", fileId, dossierId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES); } - addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet()); + addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false); } @SneakyThrows public void addToPreprocessingQueue(String dossierId, String fileId, String filename) { - var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .fileName(filename) - .build(); + var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder().dossierId(dossierId).fileId(fileId).fileName(filename).build(); setStatusPreProcessing(fileId); @@ -253,8 +262,7 @@ public class FileStatusService { @Transactional - protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, - Set sectionsToReanalyse) { + protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse, boolean triggeredManually) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var fileEntity = fileStatusPersistenceService.getStatus(fileId); @@ -264,6 +272,11 @@ public class FileStatusService { return; } + if (fileEntity.getProcessingErrorCounter() >= settings.getMaxErrorRetries() && !triggeredManually) { + log.warn("File {} was {} times retried with failure", fileEntity.getId(), fileEntity.getProcessingErrorCounter()); + return; + } + var fileModel = convert(fileEntity, FileModel.class, new FileModelMapper()); reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(fileModel, true); @@ -316,28 +329,6 @@ public class FileStatusService { } - private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, - ProcessingStatus processingStatus, FileModel fileModel) { - - if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { - return MessageType.STRUCTURE_ANALYSE; - } - if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { - return MessageType.ANALYSE; - } - if (reanalyse) { - if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed() - .isBefore(fileModel.getLastFileAttributeChange())) { - return MessageType.ANALYSE; - } - return MessageType.REANALYSE; - } else if(manualRedactionProviderService.getManualRedactions(fileId).getResizeRedactions().stream().filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed())).findFirst().isPresent()){ - return MessageType.REANALYSE; - } - return MessageType.ANALYSE; - } - - @Transactional public void setStatusReprocess(String dossierId, String fileId, boolean priority) { @@ -451,6 +442,11 @@ public class FileStatusService { return; } + if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) { + log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter()); + return; + } + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.OCR_PROCESSING); addToOcrQueue(dossierId, fileId, 2); } @@ -493,19 +489,6 @@ public class FileStatusService { } - private void addToOcrQueue(String dossierId, String fileId, int priority) { - - try { - rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> { - message.getMessageProperties().setPriority(priority); - return message; - }); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - public void updateLastOCRTime(String fileId) { fileStatusPersistenceService.updateLastOCRTime(fileId, OffsetDateTime.now()); @@ -548,8 +531,7 @@ public class FileStatusService { // wipe force redactions var forceRedactions = forceRedactionPersistenceService.findForceRedactions(fileId, false); - forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); // wipe add manual redactions var addRedactions = addRedactionPersistenceService.findAddRedactions(fileId, false); @@ -557,30 +539,108 @@ public class FileStatusService { // wipe removeRedactions var removeRedactions = removeRedactionPersistenceService.findRemoveRedactions(fileId, false); - removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); // wipe image recat var imageRecategorizations = imageRecategorizationPersistenceService.findRecategorizations(fileId, false); - imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now));// wipe image recat + imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat // wipe resize redactions var resizeRedactions = resizeRedactionPersistenceService.findResizeRedactions(fileId, false); - resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); // wipe legal basis changes var legalBasisChanges = legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false); - legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); fileStatusPersistenceService.updateHasComments(fileId, false); } - private List convertAttributes( - List fileAttributes, String dossierTemplateId) { + @Transactional + public void setStatusAnalyse(String dossierId, String fileId, boolean priority) { + + FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); + + if (fileStatus.isExcluded()) { + log.debug("File {} is excluded", fileStatus.getId()); + return; + } + + if (fileStatus.getProcessingErrorCounter() >= settings.getMaxErrorRetries()) { + log.warn("File {} was {} times retried with failure", fileStatus.getId(), fileStatus.getProcessingErrorCounter()); + return; + } + + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE); + addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet(), false); + } + + + public int countSoftDeletedFiles(String dossierId) { + + return fileStatusPersistenceService.countSoftDeletedFilesPerDossierId(dossierId); + } + + + protected void addToNerQueue(String dossierId, String fileId) { + + setStatusNerAnalyzing(fileId); + try { + rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION) + .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION) + .build()), message -> { + message.getMessageProperties().setPriority(1); + return message; + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + + private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) { + + if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { + return MessageType.STRUCTURE_ANALYSE; + } + if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { + return MessageType.ANALYSE; + } + if (reanalyse) { + if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) { + return MessageType.ANALYSE; + } + return MessageType.REANALYSE; + } else if (manualRedactionProviderService.getManualRedactions(fileId) + .getResizeRedactions() + .stream() + .filter(resize -> resize.getRequestDate().isAfter(fileModel.getLastProcessed())) + .findFirst() + .isPresent()) { + return MessageType.REANALYSE; + } + return MessageType.ANALYSE; + } + + + private void addToOcrQueue(String dossierId, String fileId, int priority) { + + try { + rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> { + message.getMessageProperties().setPriority(priority); + return message; + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + + private List convertAttributes(List fileAttributes, String dossierTemplateId) { List fileAttributeList = new ArrayList<>(); @@ -611,44 +671,4 @@ public class FileStatusService { return fileStatusPersistenceService.hasChangesSince(dossierId, since); } - - @Transactional - public void setStatusAnalyse(String dossierId, String fileId, boolean priority) { - - FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); - - if (fileStatus.isExcluded()) { - log.debug("File {} is excluded", fileStatus.getId()); - return; - } - - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.ANALYSE); - addToAnalysisQueue(dossierId, fileId, priority, Sets.newHashSet()); - } - - - protected void addToNerQueue(String dossierId, String fileId) { - - setStatusNerAnalyzing(fileId); - try { - rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION) - .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION) - .build()), message -> { - message.getMessageProperties().setPriority(1); - return message; - }); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - - public int countSoftDeletedFiles(String dossierId) { - - return fileStatusPersistenceService.countSoftDeletedFilesPerDossierId(dossierId); - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java index fb63bd836..cf117f0a5 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java @@ -30,4 +30,6 @@ public class FileManagementServiceSettings { private boolean pdf2ImageServiceEnabled; + private int maxErrorRetries = 5; + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/33-add-file-processing-error-counter-column.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/33-add-file-processing-error-counter-column.changelog.yaml new file mode 100644 index 000000000..a71bad3cc --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/33-add-file-processing-error-counter-column.changelog.yaml @@ -0,0 +1,11 @@ +databaseChangeLog: + - changeSet: + id: add-file-processing-error-counter-column + author: philipp + changes: + - addColumn: + columns: + - column: + name: processing_error_counter + type: BIGINT + tableName: file diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index d2ee74f21..4f2e1fbe9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -77,3 +77,7 @@ databaseChangeLog: file: db/changelog/sql/31-watermark-configuration.sql - include: file: db/changelog/32-added-skipped-color-type-table.changelog.yaml + - include: + file: db/changelog/33-add-file-processing-error-counter-column.changelog.yaml + - include: + file: db/changelog/sql/33-set-file-processing-error-counter.sql diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/33-set-file-processing-error-counter.sql b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/33-set-file-processing-error-counter.sql new file mode 100644 index 000000000..94ad10906 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/33-set-file-processing-error-counter.sql @@ -0,0 +1 @@ +update file set processing_error_counter = 0 where processing_error_counter is null;