diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 4f2a4e9a2..b62454f89 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -1,18 +1,17 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; -import java.time.OffsetDateTime; -import java.util.List; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; - import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; + +import java.time.OffsetDateTime; +import java.util.List; public interface FileRepository extends JpaRepository { @@ -23,25 +22,35 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId") + @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + + " e.hasHints = :hasHints, e.hasSuggestions = :hasSuggestions," + + " e.hasImages = :hasImages, e.hasUpdates = :hasUpdates, e.hasAnnotationComments = :hasComments, " + + " e.lastUpdated = :lastUpdated " + " where e.id =:fileId") void updateFlags(String fileId, OffsetDateTime lastUpdated, boolean hasRedactions, boolean hasHints, boolean hasImages, boolean hasSuggestions, boolean hasComments, boolean hasUpdates); @Modifying - @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + @Query("update FileEntity f set f.numberOfPages = :numberOfPages, f.processingStatus = :processingStatus, " + + "f.dictionaryVersion = :dictionaryVersion, f.rulesVersion = :rulesVersion, f.legalBasisVersion = :legalBasisVersion, " + + "f.analysisDuration = :analysisDuration, f.dossierDictionaryVersion = :dossierDictionaryVersion, " + + "f.analysisVersion = :analysisVersion, f.numberOfAnalyses = :analysisNumber, f.lastUpdated = :lastUpdated, " + + "f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + + "where f.id = :fileId") void updateProcessingStatus(String fileId, int numberOfPages, ProcessingStatus processingStatus, long dictionaryVersion, long rulesVersion, long legalBasisVersion, long analysisDuration, long dossierDictionaryVersion, int analysisVersion, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int analysisNumber, int processingErrorCounter); @Modifying - @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId") + @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate," + + " f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis where f.id = :fileId") void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate, boolean excludedFromAutomaticAnalysis); @Modifying - @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + " where f.id = :fileId") + @Query("update FileEntity f set f.workflowStatus = :workflowStatus, f.lastUpdated = :lastUpdated, f.approvalDate = :approvalDate " + + " where f.id = :fileId") void updateWorkflowStatus(String fileId, WorkflowStatus workflowStatus, OffsetDateTime lastUpdated, OffsetDateTime approvalDate); @@ -51,22 +60,27 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize, f.processingErrorCounter = :processingErrorCounter " + " where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + + " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize, f.processingErrorCounter = :processingErrorCounter " + + " where f.id = :fileId") void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize, int processingErrorCounter); @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.processingErrorCounter = :processingErrorCounter " + + "where f.id = :fileId") void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, int processingErrorCounter); @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + "where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.lastProcessed = :lastProcessed, f.processingErrorCounter = :processingErrorCounter " + + "where f.id = :fileId") void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastProcessed, int processingErrorCounter); @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + + "f.lastIndexed = :lastIndexed, f.processingErrorCounter = :processingErrorCounter where f.id = :fileId") void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed, int processingErrorCounter); @@ -91,17 +105,25 @@ public interface FileRepository extends JpaRepository { @Modifying - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + + "f.deleted = :softDeletedTime where f.id = :fileId") int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime softDeletedTime); @Modifying - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + "f.hardDeletedTime = :hardDeletedTime, " + "f.deleted = case " + "when f.deleted is null then :deleted " + "when f.deleted is not null then f.deleted " + "end " + "where f.id = :fileId") + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + + " f.hardDeletedTime = :hardDeletedTime, " + + " f.deleted = case " + + " when f.deleted is null then :deleted " + + " when f.deleted is not null then f.deleted " + + " end " + + " where f.id = :fileId") int setHardDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime hardDeletedTime, OffsetDateTime deleted); @Modifying - @Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " + "f.lastUpdated = :lastUpdated where f.id = :fileId") + @Query("update FileEntity f set f.assignee = :assignee, f.lastReviewer = :lastReviewer, f.lastApprover = :lastApprover, " + + "f.lastUpdated = :lastUpdated where f.id = :fileId") int setAssignee(String fileId, String assignee, String lastReviewer, String lastApprover, OffsetDateTime lastUpdated); @@ -116,22 +138,38 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, f.processingErrorCounter = 0 where f.id = :fileId") + @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.excludedFromAutomaticAnalysis = false, f.processingErrorCounter = 0 where f.id = :fileId") int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated); @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastProcessed = null," + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0 where f.id = :fileId") + @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + + "f.lastProcessed = null," + + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, " + + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = false, f.processingErrorCounter = 0 where f.id = :fileId") int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" + + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") int countSoftDeletedFilesPerDossierTemplateId(String dossierTemplateId); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.id = :dossierId" + + " and ((f.deleted is not null and f.hardDeletedTime is null) or " + + " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null") int countSoftDeletedFilesPerDossierId(String dossierId); @@ -149,7 +187,10 @@ public interface FileRepository extends JpaRepository { void setLastManualChangeDate(String fileId, OffsetDateTime lastManualChangeDate, OffsetDateTime lastUpdated); - @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries") + @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.excluded = false and f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + + " and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'ERROR' )" + + " and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + + " and f.deleted is null and f.hardDeletedTime is null and f.processingErrorCounter <= :maxRetries") List getAllRelevantStatusesForReanalysisScheduler(int maxRetries); @@ -162,19 +203,32 @@ public interface FileRepository extends JpaRepository { List getSoftDeletedFiles(List dossierIds); - @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.processingStatus ") + @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + + "inner join DossierEntity d on d.id = f.dossierId " + + "where f.deleted is null and f.hardDeletedTime is null " + + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + + "group by f.processingStatus ") List countFilesByProcessingStatus(String dossierTemplateId); - @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + "group by f.workflowStatus ") + @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + + "inner join DossierEntity d on d.id = f.dossierId " + + "where f.deleted is null and f.hardDeletedTime is null " + + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + + "group by f.workflowStatus ") List countFilesByWorkflowStatus(String dossierTemplateId); - @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) + @Query(value = "select COALESCE(sum(number_of_pages),0) as numberOfAnalyzedPages, COALESCE(sum(json_array_length(cast(excluded_pages AS json))),0) as numberOfExcludedPages " + + " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + + " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) FilePageCountsProjection countPages(String dossierTemplateId); - @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + "f.hardDeletedTime is null and f.deleted is null and " + "d.dossierTemplateId = :dossierTemplateId and " + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + + "f.hardDeletedTime is null and f.deleted is null and " + + "d.dossierTemplateId = :dossierTemplateId and " + + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") int countActiveFiles(String dossierTemplateId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java index db0a7b17f..0ba0fb3e6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationStarterService.java @@ -20,7 +20,7 @@ public class MigrationStarterService { // This should always be the latest migration version (2), but as 3.2.x does not have this, currently it is 1. // After migration to 3.3.x we must fix this. - public static final long MIGRATION_SEED_VERSION = 1; + public static final long MIGRATION_SEED_VERSION = 8; private final List migrations; private final FileManagementServiceSettings settings; diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 2c0dd8822..ce70a6146 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -89,4 +89,6 @@ databaseChangeLog: file: db/changelog/36-revert-reports-information-column.changelog.yaml - include: file: db/changelog/sql/37-update-colors.sql + - include: + file: db/changelog/sql/38-update-soft-deleted-processed-flag.sql diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/38-update-soft-deleted-processed-flag.sql b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/38-update-soft-deleted-processed-flag.sql new file mode 100644 index 000000000..1459dd088 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/38-update-soft-deleted-processed-flag.sql @@ -0,0 +1 @@ +update file set processing_status = 'PROCESSED' where deleted is not null;