From a5dea2acf189803a89dcc11c239f9cf11b908ab0 Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Tue, 10 Sep 2024 09:26:22 +0200 Subject: [PATCH] RED-9936: Clean-up job to delete all files related to hard-deleted files --- .../ApplicationConfigurationEntity.java | 3 + .../jobs/CreateJobsConfiguration.java | 27 ++- .../v1/processor/service/DossierService.java | 5 + .../service/FileDeletionService.java | 24 ++- .../service/job/DeletedFilesCleanupJob.java | 56 ++--- .../job/SoftDeletedFilesCleanupJob.java | 72 +++++++ .../DossierPersistenceService.java | 7 + .../FileStatusPersistenceService.java | 10 + .../repository/DossierRepository.java | 4 + .../repository/FileRepository.java | 9 + .../db/changelog/db.changelog-tenant.yaml | 4 +- ...try-time-to-application-configuration.yaml | 12 ++ .../tests/ApplicationConfigTest.java | 5 + .../v1/server/integration/tests/FileTest.java | 194 ++++++++++++++++-- .../AbstractPersistenceServerServiceTest.java | 8 +- .../configuration/ApplicationConfig.java | 2 + 16 files changed, 380 insertions(+), 62 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/SoftDeletedFilesCleanupJob.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/140-add-hard-delete-cleanup-retry-time-to-application-configuration.yaml diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java index 05fcc900a..e2f056b31 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/ApplicationConfigurationEntity.java @@ -32,4 +32,7 @@ public class ApplicationConfigurationEntity { @Column private int softDeleteCleanupTime = 96; + @Column + private int hardDeleteCleanupRetryTime = 72; + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/jobs/CreateJobsConfiguration.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/jobs/CreateJobsConfiguration.java index 660fbfa8f..75cf2fcd9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/jobs/CreateJobsConfiguration.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/jobs/CreateJobsConfiguration.java @@ -64,17 +64,38 @@ public class CreateJobsConfiguration { } + @Bean + public Trigger softDeletedFilesCleanupJobTrigger() throws ParseException { + + return TriggerBuilder.newTrigger() + .forJob(softDeletedFilesCleanupJobDetail()) + .withIdentity("SoftDeletedFilesCleanupJobTrigger") + .withDescription("Triggers SoftDeletedFilesCleanupJob every 30 minutes") + .withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 */30 * * * ?"))) + .build(); + } + @Bean public Trigger deletedFilesCleanupJobTrigger() throws ParseException { return TriggerBuilder.newTrigger() .forJob(deletedFilesCleanupJobDetail()) .withIdentity("DeletedFilesCleanupJobTrigger") - .withDescription("Triggers DeletedFilesCleanupJob every 30 minutes") - .withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 */30 * * * ?"))) + .withDescription("Triggers DeletedFilesCleanupJob every night at 2 AM") + .withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 0 2 * * ?"))) .build(); } + @Bean + public JobDetail softDeletedFilesCleanupJobDetail() { + + return JobBuilder.newJob() + .ofType(SoftDeletedFilesCleanupJob.class) + .storeDurably() + .withIdentity("SoftDeletedFilesCleanupJob") + .withDescription("Hard delete soft-deleted dossiers / files after certain time") + .build(); + } @Bean public JobDetail deletedFilesCleanupJobDetail() { @@ -83,7 +104,7 @@ public class CreateJobsConfiguration { .ofType(DeletedFilesCleanupJob.class) .storeDurably() .withIdentity("DeletedFilesCleanupJob") - .withDescription("Hard delete dossiers / files after certain time") + .withDescription("Delete file data and update indices of hard deleted files after certain time") .build(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java index 8a18c26d4..a4a23fd51 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java @@ -123,6 +123,11 @@ public class DossierService { dossierPersistenceService.hardDelete(dossierId); } + public void hardDeleteDossier(String dossierId, OffsetDateTime hardDeleteTime) { + + dossierPersistenceService.hardDelete(dossierId, hardDeleteTime); + } + public void undeleteDossier(String dossierId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java index 45d3564ac..d1f04c272 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java @@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService; import com.iqser.red.service.search.v1.model.IndexMessageType; + import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -131,6 +132,25 @@ public class FileDeletionService { @Transactional public void hardDeleteFiles(String dossierId, List fileIds) { + deleteAllRelatedEntries(dossierId, fileIds); + + fileStatusPersistenceService.hardDeleteFiles(fileIds); + + } + + + @Transactional + public void hardDeleteFiles(String dossierId, List fileIds, OffsetDateTime hardDeleteTime) { + + deleteAllRelatedEntries(dossierId, fileIds); + + fileStatusPersistenceService.hardDeleteFiles(fileIds, hardDeleteTime); + + } + + + private void deleteAllRelatedEntries(String dossierId, List fileIds) { + // delete all annotations addRedactionPersistenceService.deleteByFileIds(fileIds); forceRedactionPersistenceService.deleteByFileIds(fileIds); @@ -147,10 +167,6 @@ public class FileDeletionService { //delete all overrides fileIds.forEach(fileId -> componentLogMongoService.hardDeleteComponentLogEntries(dossierId, fileId)); - - - fileStatusPersistenceService.hardDeleteFiles(fileIds); - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java index 9392e8236..3c088aa8d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java @@ -3,16 +3,15 @@ package com.iqser.red.service.persistence.management.v1.processor.service.job; import java.time.OffsetDateTime; import java.util.List; -import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService; -import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; -import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils; import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantProvider; @@ -26,50 +25,35 @@ import lombok.extern.slf4j.Slf4j; @DisallowConcurrentExecution public class DeletedFilesCleanupJob implements Job { - private final DossierService dossierService; - private final FileStatusService fileStatusService; + private final FileRepository fileRepository; private final FileDeletionService fileDeletionService; private final ApplicationConfigService applicationConfigService; private final TenantProvider tenantProvider; - @Override public void execute(JobExecutionContext jobExecutionContext) { - tenantProvider.getTenants() - .forEach(tenant -> { + tenantProvider.getTenants().forEach(tenant -> { - if (!TenantUtils.isTenantReadyForPersistence(tenant)) { - return; - } + if (!TenantUtils.isTenantReadyForPersistence(tenant)) { + return; + } - TenantContext.setTenantId(tenant.getTenantId()); + TenantContext.setTenantId(tenant.getTenantId()); - var now = OffsetDateTime.now(); - List dossiers = dossierService.getAllDossiers(); - var applicationConfigurationEntity = applicationConfigService.getApplicationConfig(); + var now = OffsetDateTime.now(); + var applicationConfigurationEntity = applicationConfigService.getApplicationConfig(); - for (DossierEntity dossierEntity : dossiers) { - if (dossierEntity.getSoftDeletedTime() != null && dossierEntity.getHardDeletedTime() == null) { - if (dossierEntity.getSoftDeletedTime().isBefore(now.minusHours(applicationConfigurationEntity.getSoftDeleteCleanupTime()))) { - dossierService.hardDeleteDossier(dossierEntity.getId()); - log.info("Hard deleted dossier with dossier id {} ", dossierEntity.getId()); - } - } else { - var files = fileStatusService.getDossierStatus(dossierEntity.getId()); - for (var file : files) { - if (file.getHardDeletedTime() == null && file.getDeleted() != null && file.getDeleted() - .isBefore(now.minusHours(applicationConfigurationEntity.getSoftDeleteCleanupTime()))) { + // Directly query the files that need hard deletion + List filesToDelete = fileRepository.findFilesHardDeletedTimeAfter( + now.minusHours(applicationConfigurationEntity.getHardDeleteCleanupRetryTime())); - fileDeletionService.hardDeleteFile(file.getDossierId(), file.getId()); - fileDeletionService.hardDeleteFileDataAndIndexUpdates(dossierEntity.getId(),file.getId()); - log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), file.getId()); - } - } - } - } - - }); + for (var file : filesToDelete) { + fileDeletionService.hardDeleteFileDataAndIndexUpdates(file.getDossierId(), file.getId()); + log.info("Deleted file data and index updates for dossier id {} and file id {}", file.getDossierId(), file.getId()); + } + }); } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/SoftDeletedFilesCleanupJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/SoftDeletedFilesCleanupJob.java new file mode 100644 index 000000000..445dcdd14 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/SoftDeletedFilesCleanupJob.java @@ -0,0 +1,72 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.job; + +import java.time.OffsetDateTime; +import java.util.List; + +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils; +import com.knecon.fforesight.tenantcommons.TenantContext; +import com.knecon.fforesight.tenantcommons.TenantProvider; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +@Service +@DisallowConcurrentExecution +public class SoftDeletedFilesCleanupJob implements Job { + + private final FileRepository fileRepository; + private final DossierRepository dossierRepository; + private final DossierService dossierService; + private final FileDeletionService fileDeletionService; + private final ApplicationConfigService applicationConfigService; + private final TenantProvider tenantProvider; + + + @Override + public void execute(JobExecutionContext jobExecutionContext) { + + tenantProvider.getTenants() + .forEach(tenant -> { + + if (!TenantUtils.isTenantReadyForPersistence(tenant)) { + return; + } + + TenantContext.setTenantId(tenant.getTenantId()); + + var applicationConfigurationEntity = applicationConfigService.getApplicationConfig(); + var now = OffsetDateTime.now(); + OffsetDateTime deletionThreshold = now.minusHours(applicationConfigurationEntity.getSoftDeleteCleanupTime()); + + List dossiers = dossierRepository.findDossiersSoftDeletedBefore(deletionThreshold); + for (DossierEntity dossierEntity : dossiers) { + dossierService.hardDeleteDossier(dossierEntity.getId()); + log.info("Hard deleted dossier with dossier id {} ", dossierEntity.getId()); + + } + + List files = fileRepository.findFilesSoftDeletedBefore(deletionThreshold); + for (var file : files) { + + fileDeletionService.hardDeleteFile(file.getDossierId(), file.getId()); + log.info("Hard deleted file with dossier id {} and file id {}", file.getDossierId(), file.getId()); + + } + }); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index a4e10a712..21a03fb26 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -208,6 +208,13 @@ public class DossierPersistenceService { } + @Transactional + public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) { + + dossierRepository.hardDelete(dossierId, hardDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + } + + @Transactional public void undelete(String dossierId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 027b537e1..7fa45dd73 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -480,6 +480,14 @@ public class FileStatusPersistenceService { fileAttributesRepository.deleteByFileIds(fileIds); } + public void hardDeleteFiles(List fileIds, OffsetDateTime hardDeleteTime) { + + fileRepository.hardDeleteFiles(fileIds, ProcessingStatus.PROCESSED, hardDeleteTime); + fileAttributesRepository.deleteByFileIds(fileIds); + + + } + @Transactional public void undelete(String fileId) { @@ -678,4 +686,6 @@ public class FileStatusPersistenceService { fileRepository.setLastFlagCalculationDate(fileId, lastFlagCalculation, OffsetDateTime.now()); } + + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 6a3b41d3b..438abb196 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -106,4 +106,8 @@ public interface DossierRepository extends JpaRepository @Query("select d.dossierTemplateId from DossierEntity d where d.id = :dossierId") String findDossierTemplateId(@Param("dossierId") String dossierId); + + @Query("SELECT d FROM DossierEntity d WHERE d.hardDeletedTime IS NULL AND d.softDeletedTime IS NOT NULL AND d.softDeletedTime < :deletionThreshold") + List findDossiersSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 9ec7f1e5e..78f72a8ab 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -313,6 +313,7 @@ public interface FileRepository extends JpaRepository { @Query("select f from FileEntity f where f.hardDeletedTime is not null") List getHardDeletedFiles(); + @Query("select f from FileEntity f where f.processingStatus = 'ERROR' and f.deleted is null and f.hardDeletedTime is null ") List getAllErrorFilesExcludeDeleted(); @@ -405,6 +406,14 @@ public interface FileRepository extends JpaRepository { + "where f.id in (:fileIds)") int hardDeleteFiles(@Param("fileIds") List fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime); + + @Query("SELECT f FROM FileEntity f WHERE f.hardDeletedTime IS NOT NULL AND f.hardDeletedTime > :deletionThreshold") + List findFilesHardDeletedTimeAfter(@Param("deletionThreshold") OffsetDateTime deletionThreshold); + + + @Query("SELECT f FROM FileEntity f WHERE f.hardDeletedTime IS NULL AND f.deleted IS NOT NULL AND f.deleted < :deletionThreshold") + List findFilesSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml index ce6383695..4dfb13b5b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml @@ -220,4 +220,6 @@ databaseChangeLog: - include: file: db/changelog/tenant/133-add-technical-name-to-legal_basis.yaml - include: - file: db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml \ No newline at end of file + file: db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml + - include: + file: db/changelog/tenant/140-add-hard-delete-cleanup-retry-time-to-application-configuration.yaml \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/140-add-hard-delete-cleanup-retry-time-to-application-configuration.yaml b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/140-add-hard-delete-cleanup-retry-time-to-application-configuration.yaml new file mode 100644 index 000000000..32cf77666 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/140-add-hard-delete-cleanup-retry-time-to-application-configuration.yaml @@ -0,0 +1,12 @@ +databaseChangeLog: + - changeSet: + id: add-hard-delete-cleanup-time-to-application-configuration + author: maverick + changes: + - addColumn: + columns: + - column: + name: hard_delete_cleanup_retry_time + type: INTEGER + defaultValue: '72' + tableName: application_configuration diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApplicationConfigTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApplicationConfigTest.java index 79b6ff16c..63ce356bb 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApplicationConfigTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ApplicationConfigTest.java @@ -22,24 +22,29 @@ public class ApplicationConfigTest extends AbstractPersistenceServerServiceTest .downloadCleanupDownloadFilesHours(8) .downloadCleanupNotDownloadFilesHours(72) .softDeleteCleanupTime(96) + .hardDeleteCleanupRetryTime(72) .build(); var result = appConfigClient.createOrUpdateAppConfig(appConfig); assertThat(result.getSoftDeleteCleanupTime()).isEqualTo(appConfig.getSoftDeleteCleanupTime()); + assertThat(result.getHardDeleteCleanupRetryTime()).isEqualTo(appConfig.getHardDeleteCleanupRetryTime()); assertThat(result.getDownloadCleanupDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupDownloadFilesHours()); assertThat(result.getDownloadCleanupNotDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupNotDownloadFilesHours()); var getResult = appConfigClient.getCurrentApplicationConfig(); assertThat(getResult.getSoftDeleteCleanupTime()).isEqualTo(appConfig.getSoftDeleteCleanupTime()); + assertThat(getResult.getHardDeleteCleanupRetryTime()).isEqualTo(appConfig.getHardDeleteCleanupRetryTime()); assertThat(getResult.getDownloadCleanupDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupDownloadFilesHours()); assertThat(getResult.getDownloadCleanupNotDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupNotDownloadFilesHours()); appConfig.setDownloadCleanupDownloadFilesHours(16); appConfig.setDownloadCleanupNotDownloadFilesHours(80); appConfig.setSoftDeleteCleanupTime(100); + appConfig.setHardDeleteCleanupRetryTime(100); appConfigClient.createOrUpdateAppConfig(appConfig); getResult = appConfigClient.getCurrentApplicationConfig(); assertThat(getResult.getSoftDeleteCleanupTime()).isEqualTo(appConfig.getSoftDeleteCleanupTime()); + assertThat(getResult.getHardDeleteCleanupRetryTime()).isEqualTo(appConfig.getHardDeleteCleanupRetryTime()); assertThat(getResult.getDownloadCleanupDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupDownloadFilesHours()); assertThat(getResult.getDownloadCleanupNotDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupNotDownloadFilesHours()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 0d6cae8a6..82b723762 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -1,6 +1,8 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.any; @@ -14,6 +16,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.junit.jupiter.api.Assertions; @@ -27,7 +30,6 @@ import com.google.common.collect.Sets; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient; import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; @@ -43,8 +45,12 @@ import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvid import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; +import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; +import com.iqser.red.service.persistence.management.v1.processor.service.job.DeletedFilesCleanupJob; +import com.iqser.red.service.persistence.management.v1.processor.service.job.SoftDeletedFilesCleanupJob; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes; @@ -62,6 +68,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -72,6 +79,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Forc import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel; +import com.knecon.fforesight.tenantcommons.model.TenantResponse; import feign.FeignException; import lombok.SneakyThrows; @@ -111,6 +119,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest { @Autowired private FileAttributeConfigClient fileAttributeConfigClient; + @Autowired + private DossierService dossierService; + + @Autowired + private FileDeletionService fileDeletionService; + @Autowired private FileManagementClient fileManagementClient; @@ -120,15 +134,18 @@ public class FileTest extends AbstractPersistenceServerServiceTest { @Autowired private UserProvider userProvider; - @Autowired - private DossierTemplateClient dossierTemplateClient; - @Autowired private FileManagementStorageService fileManagementStorageService; @Autowired private FileStatusService fileStatusService; + @Autowired + private DeletedFilesCleanupJob deletedFilesCleanupJob; + + @Autowired + private SoftDeletedFilesCleanupJob softDeletedFilesCleanupJob; + @Test public void testFileSoftDeleteReupload() { @@ -771,16 +788,16 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); var recatResponse = manualRedactionClient.recategorizeBulk(dossierId, - fileId, - Set.of(RecategorizationRequestModel.builder() - .annotationId(annotation2Id) - .comment("comment edit via edit dialog") - .type(type.getType()) - .legalBasis("") - .section("section") - .value("value entry 3") - .build()), - false); + fileId, + Set.of(RecategorizationRequestModel.builder() + .annotationId(annotation2Id) + .comment("comment edit via edit dialog") + .type(type.getType()) + .legalBasis("") + .section("section") + .value("value entry 3") + .build()), + false); var loadedFile = fileClient.getFileStatus(dossierId, fileId); @@ -791,7 +808,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(annotationComments.getComments()).hasSize(1); AnnotationComments annotationCommentsForInitialAnnotation = manualRedactionClient.getComments(dossierId, fileId, annotation2Id); assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0); - AnnotationComments annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, fileId, recatResponse.getManualAddResponses().get(0).getAnnotationId()); + AnnotationComments annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, + fileId, + recatResponse.getManualAddResponses() + .get(0).getAnnotationId()); assertThat(annotationCommentsForManualRecat.getComments()).hasSize(1); //overwrite the file @@ -807,9 +827,151 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(annotationComments.getComments()).hasSize(0); annotationCommentsForInitialAnnotation = manualRedactionClient.getComments(dossierId, fileId, annotation2Id); assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0); - annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, fileId, recatResponse.getManualAddResponses().get(0).getAnnotationId()); + annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, + fileId, + recatResponse.getManualAddResponses() + .get(0).getAnnotationId()); assertThat(annotationCommentsForManualRecat.getComments()).hasSize(0); } + + @Test + public void testSoftDeletedFilesCleanupJob() { + + OffsetDateTime enoughTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getSoftDeleteCleanupTime() / 24); + + FileModel fileModel = executeSoftDeletedFilesCleanupJobForGivenTime(enoughTimePassed); + + assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getHardDeletedTime()); + assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getSoftDeletedTime()); + assertNotNull(fileModel.getHardDeletedTime()); + assertNotNull(fileModel.getDeleted()); + + } + + + @Test + public void testSoftDeletedFilesCleanupJobUnsuccessful() { + + OffsetDateTime notEnoughTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getSoftDeleteCleanupTime() / 24).plusHours(1); + + FileModel fileModel = executeSoftDeletedFilesCleanupJobForGivenTime(notEnoughTimePassed); + + assertNull(dossierService.getDossierById(fileModel.getDossierId()).getHardDeletedTime()); + assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getSoftDeletedTime()); + assertNull(fileModel.getHardDeletedTime()); + assertNotNull(fileModel.getDeleted()); + } + + + private FileModel executeSoftDeletedFilesCleanupJobForGivenTime(OffsetDateTime offsetDateTime) { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + String dossierId = dossier.getId(); + + var file = fileTesterAndProvider.testAndProvideFile(dossier); + String fileId = file.getId(); + + var otherFile = fileTesterAndProvider.testAndProvideFile(dossier, "otherFile"); + String otherFileId = otherFile.getId(); + + dossierService.softDeleteDossier(dossierId, offsetDateTime); + fileDeletionService.softDeleteFiles(dossierId, List.of(fileId, otherFileId), offsetDateTime); + fileDeletionService.hardDeleteFile(dossierId, otherFileId); + + when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build())); + + assertNull(dossierService.getDossierById(dossierId).getHardDeletedTime()); + assertNotNull(dossierService.getDossierById(dossierId).getSoftDeletedTime()); + assertNull(fileStatusService.getDossierStatus(dossierId) + .get(0).getHardDeletedTime()); + assertNotNull(fileStatusService.getDossierStatus(dossierId) + .get(0).getDeleted()); + assertNotNull(fileStatusService.getDossierStatus(dossierId) + .get(1).getHardDeletedTime()); + assertNotNull(fileStatusService.getDossierStatus(dossierId) + .get(1).getDeleted()); + + softDeletedFilesCleanupJob.execute(null); + + Optional otherFileModel = fileStatusService.getDossierStatus(dossierId) + .stream() + .filter(fm -> fm.getId().equals(otherFileId)) + .findFirst(); + Assertions.assertTrue(otherFileModel.isPresent()); + + assertNotNull(otherFileModel.get().getHardDeletedTime()); + assertNotNull(otherFileModel.get().getDeleted()); + + + Optional fileModel = fileStatusService.getDossierStatus(dossierId) + .stream() + .filter(fm -> fm.getId().equals(fileId)) + .findFirst(); + Assertions.assertTrue(fileModel.isPresent()); + return fileModel.get(); + } + + + @Test + public void testHardDeletedFilesCleanupJob() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + String dossierId = dossier.getId(); + + var file = fileTesterAndProvider.testAndProvideFile(dossier); + String fileId = file.getId(); + + OffsetDateTime withinRetryTime = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getHardDeleteCleanupRetryTime() / 24).plusHours(1); + + executeDeletedFilesCleanupJobForGivenTime(withinRetryTime, dossierId, fileId); + + Assertions.assertFalse(fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)); + + } + + + @Test + public void testHardDeletedFilesCleanupJobUnsuccessful() { + + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + String dossierId = dossier.getId(); + + var file = fileTesterAndProvider.testAndProvideFile(dossier); + String fileId = file.getId(); + + OffsetDateTime tooMuchTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getHardDeleteCleanupRetryTime() / 24).minusHours(1); + + executeDeletedFilesCleanupJobForGivenTime(tooMuchTimePassed, dossierId, fileId); + + Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN)); + + } + + + private void executeDeletedFilesCleanupJobForGivenTime(OffsetDateTime offsetDateTime, String dossierId, String fileId) { + + byte[] bytesToStore = {1, 2, 3, 4}; + fileManagementStorageService.storeObject(dossierId, fileId, FileType.ORIGIN, new ByteArrayInputStream(bytesToStore)); + + dossierService.hardDeleteDossier(dossierId, offsetDateTime); + fileDeletionService.hardDeleteFiles(dossierId, List.of(fileId), offsetDateTime); + + when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build())); + + assertNotNull(dossierService.getDossierById(dossierId).getHardDeletedTime()); + assertNotNull(fileStatusService.getDossierStatus(dossierId) + .get(0).getHardDeletedTime()); + Assertions.assertArrayEquals(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN), bytesToStore); + + deletedFilesCleanupJob.execute(null); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index 666156136..790a2f4bb 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -22,7 +22,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.quartz.Scheduler; -import org.springframework.amqp.core.AmqpAdmin; import org.springframework.amqp.rabbit.core.RabbitAdmin; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry; @@ -311,7 +310,12 @@ public abstract class AbstractPersistenceServerServiceTest { TenantContext.setTenantId(TENANT_1); - ApplicationConfig appConfig = ApplicationConfig.builder().downloadCleanupDownloadFilesHours(8).downloadCleanupNotDownloadFilesHours(72).softDeleteCleanupTime(96).build(); + ApplicationConfig appConfig = ApplicationConfig.builder() + .downloadCleanupDownloadFilesHours(8) + .downloadCleanupNotDownloadFilesHours(72) + .softDeleteCleanupTime(96) + .hardDeleteCleanupRetryTime(72) + .build(); applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class)); tokenService.setUser("manageradmin1@test.com", "secret"); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/configuration/ApplicationConfig.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/configuration/ApplicationConfig.java index 773e222ea..904dda344 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/configuration/ApplicationConfig.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/configuration/ApplicationConfig.java @@ -18,5 +18,7 @@ public class ApplicationConfig { private int downloadCleanupNotDownloadFilesHours; @Min(1) private int softDeleteCleanupTime; + @Min(1) + private int hardDeleteCleanupRetryTime; }