RED-9936: Clean-up job to delete all files related to hard-deleted files
This commit is contained in:
Maverick Studer
parent
9143292e0f
commit
a5dea2acf1
@ -32,4 +32,7 @@ public class ApplicationConfigurationEntity {
|
||||
@Column
|
||||
private int softDeleteCleanupTime = 96;
|
||||
|
||||
@Column
|
||||
private int hardDeleteCleanupRetryTime = 72;
|
||||
|
||||
}
|
||||
|
||||
@ -64,17 +64,38 @@ public class CreateJobsConfiguration {
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Trigger softDeletedFilesCleanupJobTrigger() throws ParseException {
|
||||
|
||||
return TriggerBuilder.newTrigger()
|
||||
.forJob(softDeletedFilesCleanupJobDetail())
|
||||
.withIdentity("SoftDeletedFilesCleanupJobTrigger")
|
||||
.withDescription("Triggers SoftDeletedFilesCleanupJob every 30 minutes")
|
||||
.withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 */30 * * * ?")))
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Trigger deletedFilesCleanupJobTrigger() throws ParseException {
|
||||
|
||||
return TriggerBuilder.newTrigger()
|
||||
.forJob(deletedFilesCleanupJobDetail())
|
||||
.withIdentity("DeletedFilesCleanupJobTrigger")
|
||||
.withDescription("Triggers DeletedFilesCleanupJob every 30 minutes")
|
||||
.withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 */30 * * * ?")))
|
||||
.withDescription("Triggers DeletedFilesCleanupJob every night at 2 AM")
|
||||
.withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 0 2 * * ?")))
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public JobDetail softDeletedFilesCleanupJobDetail() {
|
||||
|
||||
return JobBuilder.newJob()
|
||||
.ofType(SoftDeletedFilesCleanupJob.class)
|
||||
.storeDurably()
|
||||
.withIdentity("SoftDeletedFilesCleanupJob")
|
||||
.withDescription("Hard delete soft-deleted dossiers / files after certain time")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public JobDetail deletedFilesCleanupJobDetail() {
|
||||
@ -83,7 +104,7 @@ public class CreateJobsConfiguration {
|
||||
.ofType(DeletedFilesCleanupJob.class)
|
||||
.storeDurably()
|
||||
.withIdentity("DeletedFilesCleanupJob")
|
||||
.withDescription("Hard delete dossiers / files after certain time")
|
||||
.withDescription("Delete file data and update indices of hard deleted files after certain time")
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@ -123,6 +123,11 @@ public class DossierService {
|
||||
dossierPersistenceService.hardDelete(dossierId);
|
||||
}
|
||||
|
||||
public void hardDeleteDossier(String dossierId, OffsetDateTime hardDeleteTime) {
|
||||
|
||||
dossierPersistenceService.hardDelete(dossierId, hardDeleteTime);
|
||||
}
|
||||
|
||||
|
||||
public void undeleteDossier(String dossierId) {
|
||||
|
||||
|
||||
@ -19,6 +19,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
|
||||
import com.iqser.red.service.search.v1.model.IndexMessageType;
|
||||
|
||||
import jakarta.transaction.Transactional;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -131,6 +132,25 @@ public class FileDeletionService {
|
||||
@Transactional
|
||||
public void hardDeleteFiles(String dossierId, List<String> fileIds) {
|
||||
|
||||
deleteAllRelatedEntries(dossierId, fileIds);
|
||||
|
||||
fileStatusPersistenceService.hardDeleteFiles(fileIds);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void hardDeleteFiles(String dossierId, List<String> fileIds, OffsetDateTime hardDeleteTime) {
|
||||
|
||||
deleteAllRelatedEntries(dossierId, fileIds);
|
||||
|
||||
fileStatusPersistenceService.hardDeleteFiles(fileIds, hardDeleteTime);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void deleteAllRelatedEntries(String dossierId, List<String> fileIds) {
|
||||
|
||||
// delete all annotations
|
||||
addRedactionPersistenceService.deleteByFileIds(fileIds);
|
||||
forceRedactionPersistenceService.deleteByFileIds(fileIds);
|
||||
@ -147,10 +167,6 @@ public class FileDeletionService {
|
||||
|
||||
//delete all overrides
|
||||
fileIds.forEach(fileId -> componentLogMongoService.hardDeleteComponentLogEntries(dossierId, fileId));
|
||||
|
||||
|
||||
fileStatusPersistenceService.hardDeleteFiles(fileIds);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -3,16 +3,15 @@ package com.iqser.red.service.persistence.management.v1.processor.service.job;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService;
|
||||
import org.quartz.DisallowConcurrentExecution;
|
||||
import org.quartz.Job;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
import com.knecon.fforesight.tenantcommons.TenantProvider;
|
||||
@ -26,50 +25,35 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@DisallowConcurrentExecution
|
||||
public class DeletedFilesCleanupJob implements Job {
|
||||
|
||||
private final DossierService dossierService;
|
||||
private final FileStatusService fileStatusService;
|
||||
private final FileRepository fileRepository;
|
||||
private final FileDeletionService fileDeletionService;
|
||||
private final ApplicationConfigService applicationConfigService;
|
||||
private final TenantProvider tenantProvider;
|
||||
|
||||
|
||||
@Override
|
||||
public void execute(JobExecutionContext jobExecutionContext) {
|
||||
|
||||
tenantProvider.getTenants()
|
||||
.forEach(tenant -> {
|
||||
tenantProvider.getTenants().forEach(tenant -> {
|
||||
|
||||
if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
|
||||
return;
|
||||
}
|
||||
if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
|
||||
return;
|
||||
}
|
||||
|
||||
TenantContext.setTenantId(tenant.getTenantId());
|
||||
TenantContext.setTenantId(tenant.getTenantId());
|
||||
|
||||
var now = OffsetDateTime.now();
|
||||
List<DossierEntity> dossiers = dossierService.getAllDossiers();
|
||||
var applicationConfigurationEntity = applicationConfigService.getApplicationConfig();
|
||||
var now = OffsetDateTime.now();
|
||||
var applicationConfigurationEntity = applicationConfigService.getApplicationConfig();
|
||||
|
||||
for (DossierEntity dossierEntity : dossiers) {
|
||||
if (dossierEntity.getSoftDeletedTime() != null && dossierEntity.getHardDeletedTime() == null) {
|
||||
if (dossierEntity.getSoftDeletedTime().isBefore(now.minusHours(applicationConfigurationEntity.getSoftDeleteCleanupTime()))) {
|
||||
dossierService.hardDeleteDossier(dossierEntity.getId());
|
||||
log.info("Hard deleted dossier with dossier id {} ", dossierEntity.getId());
|
||||
}
|
||||
} else {
|
||||
var files = fileStatusService.getDossierStatus(dossierEntity.getId());
|
||||
for (var file : files) {
|
||||
if (file.getHardDeletedTime() == null && file.getDeleted() != null && file.getDeleted()
|
||||
.isBefore(now.minusHours(applicationConfigurationEntity.getSoftDeleteCleanupTime()))) {
|
||||
// Directly query the files that need hard deletion
|
||||
List<FileEntity> filesToDelete = fileRepository.findFilesHardDeletedTimeAfter(
|
||||
now.minusHours(applicationConfigurationEntity.getHardDeleteCleanupRetryTime()));
|
||||
|
||||
fileDeletionService.hardDeleteFile(file.getDossierId(), file.getId());
|
||||
fileDeletionService.hardDeleteFileDataAndIndexUpdates(dossierEntity.getId(),file.getId());
|
||||
log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), file.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
for (var file : filesToDelete) {
|
||||
fileDeletionService.hardDeleteFileDataAndIndexUpdates(file.getDossierId(), file.getId());
|
||||
log.info("Deleted file data and index updates for dossier id {} and file id {}", file.getDossierId(), file.getId());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,72 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.job;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import org.quartz.DisallowConcurrentExecution;
|
||||
import org.quartz.Job;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.TenantUtils;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
import com.knecon.fforesight.tenantcommons.TenantProvider;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@Service
|
||||
@DisallowConcurrentExecution
|
||||
public class SoftDeletedFilesCleanupJob implements Job {
|
||||
|
||||
private final FileRepository fileRepository;
|
||||
private final DossierRepository dossierRepository;
|
||||
private final DossierService dossierService;
|
||||
private final FileDeletionService fileDeletionService;
|
||||
private final ApplicationConfigService applicationConfigService;
|
||||
private final TenantProvider tenantProvider;
|
||||
|
||||
|
||||
@Override
|
||||
public void execute(JobExecutionContext jobExecutionContext) {
|
||||
|
||||
tenantProvider.getTenants()
|
||||
.forEach(tenant -> {
|
||||
|
||||
if (!TenantUtils.isTenantReadyForPersistence(tenant)) {
|
||||
return;
|
||||
}
|
||||
|
||||
TenantContext.setTenantId(tenant.getTenantId());
|
||||
|
||||
var applicationConfigurationEntity = applicationConfigService.getApplicationConfig();
|
||||
var now = OffsetDateTime.now();
|
||||
OffsetDateTime deletionThreshold = now.minusHours(applicationConfigurationEntity.getSoftDeleteCleanupTime());
|
||||
|
||||
List<DossierEntity> dossiers = dossierRepository.findDossiersSoftDeletedBefore(deletionThreshold);
|
||||
for (DossierEntity dossierEntity : dossiers) {
|
||||
dossierService.hardDeleteDossier(dossierEntity.getId());
|
||||
log.info("Hard deleted dossier with dossier id {} ", dossierEntity.getId());
|
||||
|
||||
}
|
||||
|
||||
List<FileEntity> files = fileRepository.findFilesSoftDeletedBefore(deletionThreshold);
|
||||
for (var file : files) {
|
||||
|
||||
fileDeletionService.hardDeleteFile(file.getDossierId(), file.getId());
|
||||
log.info("Hard deleted file with dossier id {} and file id {}", file.getDossierId(), file.getId());
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@ -208,6 +208,13 @@ public class DossierPersistenceService {
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void hardDelete(String dossierId, OffsetDateTime hardDeletedTime) {
|
||||
|
||||
dossierRepository.hardDelete(dossierId, hardDeletedTime, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void undelete(String dossierId) {
|
||||
|
||||
|
||||
@ -480,6 +480,14 @@ public class FileStatusPersistenceService {
|
||||
fileAttributesRepository.deleteByFileIds(fileIds);
|
||||
}
|
||||
|
||||
public void hardDeleteFiles(List<String> fileIds, OffsetDateTime hardDeleteTime) {
|
||||
|
||||
fileRepository.hardDeleteFiles(fileIds, ProcessingStatus.PROCESSED, hardDeleteTime);
|
||||
fileAttributesRepository.deleteByFileIds(fileIds);
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void undelete(String fileId) {
|
||||
@ -678,4 +686,6 @@ public class FileStatusPersistenceService {
|
||||
fileRepository.setLastFlagCalculationDate(fileId, lastFlagCalculation, OffsetDateTime.now());
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -106,4 +106,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
|
||||
@Query("select d.dossierTemplateId from DossierEntity d where d.id = :dossierId")
|
||||
String findDossierTemplateId(@Param("dossierId") String dossierId);
|
||||
|
||||
|
||||
@Query("SELECT d FROM DossierEntity d WHERE d.hardDeletedTime IS NULL AND d.softDeletedTime IS NOT NULL AND d.softDeletedTime < :deletionThreshold")
|
||||
List<DossierEntity> findDossiersSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold);
|
||||
|
||||
}
|
||||
|
||||
@ -313,6 +313,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
@Query("select f from FileEntity f where f.hardDeletedTime is not null")
|
||||
List<FileEntity> getHardDeletedFiles();
|
||||
|
||||
|
||||
@Query("select f from FileEntity f where f.processingStatus = 'ERROR' and f.deleted is null and f.hardDeletedTime is null ")
|
||||
List<FileEntity> getAllErrorFilesExcludeDeleted();
|
||||
|
||||
@ -405,6 +406,14 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
+ "where f.id in (:fileIds)")
|
||||
int hardDeleteFiles(@Param("fileIds") List<String> fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime);
|
||||
|
||||
|
||||
@Query("SELECT f FROM FileEntity f WHERE f.hardDeletedTime IS NOT NULL AND f.hardDeletedTime > :deletionThreshold")
|
||||
List<FileEntity> findFilesHardDeletedTimeAfter(@Param("deletionThreshold") OffsetDateTime deletionThreshold);
|
||||
|
||||
|
||||
@Query("SELECT f FROM FileEntity f WHERE f.hardDeletedTime IS NULL AND f.deleted IS NOT NULL AND f.deleted < :deletionThreshold")
|
||||
List<FileEntity> findFilesSoftDeletedBefore(@Param("deletionThreshold") OffsetDateTime deletionThreshold);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -220,4 +220,6 @@ databaseChangeLog:
|
||||
- include:
|
||||
file: db/changelog/tenant/133-add-technical-name-to-legal_basis.yaml
|
||||
- include:
|
||||
file: db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml
|
||||
file: db/changelog/tenant/139-add-based-on-dict-annotation-id-to-manual_force_changes.yaml
|
||||
- include:
|
||||
file: db/changelog/tenant/140-add-hard-delete-cleanup-retry-time-to-application-configuration.yaml
|
||||
@ -0,0 +1,12 @@
|
||||
databaseChangeLog:
|
||||
- changeSet:
|
||||
id: add-hard-delete-cleanup-time-to-application-configuration
|
||||
author: maverick
|
||||
changes:
|
||||
- addColumn:
|
||||
columns:
|
||||
- column:
|
||||
name: hard_delete_cleanup_retry_time
|
||||
type: INTEGER
|
||||
defaultValue: '72'
|
||||
tableName: application_configuration
|
||||
@ -22,24 +22,29 @@ public class ApplicationConfigTest extends AbstractPersistenceServerServiceTest
|
||||
.downloadCleanupDownloadFilesHours(8)
|
||||
.downloadCleanupNotDownloadFilesHours(72)
|
||||
.softDeleteCleanupTime(96)
|
||||
.hardDeleteCleanupRetryTime(72)
|
||||
.build();
|
||||
var result = appConfigClient.createOrUpdateAppConfig(appConfig);
|
||||
|
||||
assertThat(result.getSoftDeleteCleanupTime()).isEqualTo(appConfig.getSoftDeleteCleanupTime());
|
||||
assertThat(result.getHardDeleteCleanupRetryTime()).isEqualTo(appConfig.getHardDeleteCleanupRetryTime());
|
||||
assertThat(result.getDownloadCleanupDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupDownloadFilesHours());
|
||||
assertThat(result.getDownloadCleanupNotDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupNotDownloadFilesHours());
|
||||
var getResult = appConfigClient.getCurrentApplicationConfig();
|
||||
assertThat(getResult.getSoftDeleteCleanupTime()).isEqualTo(appConfig.getSoftDeleteCleanupTime());
|
||||
assertThat(getResult.getHardDeleteCleanupRetryTime()).isEqualTo(appConfig.getHardDeleteCleanupRetryTime());
|
||||
assertThat(getResult.getDownloadCleanupDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupDownloadFilesHours());
|
||||
assertThat(getResult.getDownloadCleanupNotDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupNotDownloadFilesHours());
|
||||
|
||||
appConfig.setDownloadCleanupDownloadFilesHours(16);
|
||||
appConfig.setDownloadCleanupNotDownloadFilesHours(80);
|
||||
appConfig.setSoftDeleteCleanupTime(100);
|
||||
appConfig.setHardDeleteCleanupRetryTime(100);
|
||||
|
||||
appConfigClient.createOrUpdateAppConfig(appConfig);
|
||||
getResult = appConfigClient.getCurrentApplicationConfig();
|
||||
assertThat(getResult.getSoftDeleteCleanupTime()).isEqualTo(appConfig.getSoftDeleteCleanupTime());
|
||||
assertThat(getResult.getHardDeleteCleanupRetryTime()).isEqualTo(appConfig.getHardDeleteCleanupRetryTime());
|
||||
assertThat(getResult.getDownloadCleanupDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupDownloadFilesHours());
|
||||
assertThat(getResult.getDownloadCleanupNotDownloadFilesHours()).isEqualTo(appConfig.getDownloadCleanupNotDownloadFilesHours());
|
||||
}
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
package com.iqser.red.service.peristence.v1.server.integration.tests;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.Mockito.any;
|
||||
@ -14,6 +16,7 @@ import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
@ -27,7 +30,6 @@ import com.google.common.collect.Sets;
|
||||
import com.google.common.hash.HashFunction;
|
||||
import com.google.common.hash.Hashing;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
|
||||
@ -43,8 +45,12 @@ import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvid
|
||||
import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider;
|
||||
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.job.DeletedFilesCleanupJob;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.job.SoftDeletedFilesCleanupJob;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
|
||||
@ -62,6 +68,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
|
||||
@ -72,6 +79,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.Forc
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
|
||||
import com.knecon.fforesight.tenantcommons.model.TenantResponse;
|
||||
|
||||
import feign.FeignException;
|
||||
import lombok.SneakyThrows;
|
||||
@ -111,6 +119,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
|
||||
@Autowired
|
||||
private FileAttributeConfigClient fileAttributeConfigClient;
|
||||
|
||||
@Autowired
|
||||
private DossierService dossierService;
|
||||
|
||||
@Autowired
|
||||
private FileDeletionService fileDeletionService;
|
||||
|
||||
@Autowired
|
||||
private FileManagementClient fileManagementClient;
|
||||
|
||||
@ -120,15 +134,18 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
|
||||
@Autowired
|
||||
private UserProvider userProvider;
|
||||
|
||||
@Autowired
|
||||
private DossierTemplateClient dossierTemplateClient;
|
||||
|
||||
@Autowired
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
@Autowired
|
||||
private FileStatusService fileStatusService;
|
||||
|
||||
@Autowired
|
||||
private DeletedFilesCleanupJob deletedFilesCleanupJob;
|
||||
|
||||
@Autowired
|
||||
private SoftDeletedFilesCleanupJob softDeletedFilesCleanupJob;
|
||||
|
||||
|
||||
@Test
|
||||
public void testFileSoftDeleteReupload() {
|
||||
@ -771,16 +788,16 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
|
||||
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
|
||||
|
||||
var recatResponse = manualRedactionClient.recategorizeBulk(dossierId,
|
||||
fileId,
|
||||
Set.of(RecategorizationRequestModel.builder()
|
||||
.annotationId(annotation2Id)
|
||||
.comment("comment edit via edit dialog")
|
||||
.type(type.getType())
|
||||
.legalBasis("")
|
||||
.section("section")
|
||||
.value("value entry 3")
|
||||
.build()),
|
||||
false);
|
||||
fileId,
|
||||
Set.of(RecategorizationRequestModel.builder()
|
||||
.annotationId(annotation2Id)
|
||||
.comment("comment edit via edit dialog")
|
||||
.type(type.getType())
|
||||
.legalBasis("")
|
||||
.section("section")
|
||||
.value("value entry 3")
|
||||
.build()),
|
||||
false);
|
||||
|
||||
var loadedFile = fileClient.getFileStatus(dossierId, fileId);
|
||||
|
||||
@ -791,7 +808,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
|
||||
assertThat(annotationComments.getComments()).hasSize(1);
|
||||
AnnotationComments annotationCommentsForInitialAnnotation = manualRedactionClient.getComments(dossierId, fileId, annotation2Id);
|
||||
assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0);
|
||||
AnnotationComments annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, fileId, recatResponse.getManualAddResponses().get(0).getAnnotationId());
|
||||
AnnotationComments annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId,
|
||||
fileId,
|
||||
recatResponse.getManualAddResponses()
|
||||
.get(0).getAnnotationId());
|
||||
assertThat(annotationCommentsForManualRecat.getComments()).hasSize(1);
|
||||
|
||||
//overwrite the file
|
||||
@ -807,9 +827,151 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
|
||||
assertThat(annotationComments.getComments()).hasSize(0);
|
||||
annotationCommentsForInitialAnnotation = manualRedactionClient.getComments(dossierId, fileId, annotation2Id);
|
||||
assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0);
|
||||
annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId, fileId, recatResponse.getManualAddResponses().get(0).getAnnotationId());
|
||||
annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId,
|
||||
fileId,
|
||||
recatResponse.getManualAddResponses()
|
||||
.get(0).getAnnotationId());
|
||||
assertThat(annotationCommentsForManualRecat.getComments()).hasSize(0);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSoftDeletedFilesCleanupJob() {
|
||||
|
||||
OffsetDateTime enoughTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getSoftDeleteCleanupTime() / 24);
|
||||
|
||||
FileModel fileModel = executeSoftDeletedFilesCleanupJobForGivenTime(enoughTimePassed);
|
||||
|
||||
assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getHardDeletedTime());
|
||||
assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getSoftDeletedTime());
|
||||
assertNotNull(fileModel.getHardDeletedTime());
|
||||
assertNotNull(fileModel.getDeleted());
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSoftDeletedFilesCleanupJobUnsuccessful() {
|
||||
|
||||
OffsetDateTime notEnoughTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getSoftDeleteCleanupTime() / 24).plusHours(1);
|
||||
|
||||
FileModel fileModel = executeSoftDeletedFilesCleanupJobForGivenTime(notEnoughTimePassed);
|
||||
|
||||
assertNull(dossierService.getDossierById(fileModel.getDossierId()).getHardDeletedTime());
|
||||
assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getSoftDeletedTime());
|
||||
assertNull(fileModel.getHardDeletedTime());
|
||||
assertNotNull(fileModel.getDeleted());
|
||||
}
|
||||
|
||||
|
||||
private FileModel executeSoftDeletedFilesCleanupJobForGivenTime(OffsetDateTime offsetDateTime) {
|
||||
|
||||
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
|
||||
|
||||
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
|
||||
String dossierId = dossier.getId();
|
||||
|
||||
var file = fileTesterAndProvider.testAndProvideFile(dossier);
|
||||
String fileId = file.getId();
|
||||
|
||||
var otherFile = fileTesterAndProvider.testAndProvideFile(dossier, "otherFile");
|
||||
String otherFileId = otherFile.getId();
|
||||
|
||||
dossierService.softDeleteDossier(dossierId, offsetDateTime);
|
||||
fileDeletionService.softDeleteFiles(dossierId, List.of(fileId, otherFileId), offsetDateTime);
|
||||
fileDeletionService.hardDeleteFile(dossierId, otherFileId);
|
||||
|
||||
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
|
||||
|
||||
assertNull(dossierService.getDossierById(dossierId).getHardDeletedTime());
|
||||
assertNotNull(dossierService.getDossierById(dossierId).getSoftDeletedTime());
|
||||
assertNull(fileStatusService.getDossierStatus(dossierId)
|
||||
.get(0).getHardDeletedTime());
|
||||
assertNotNull(fileStatusService.getDossierStatus(dossierId)
|
||||
.get(0).getDeleted());
|
||||
assertNotNull(fileStatusService.getDossierStatus(dossierId)
|
||||
.get(1).getHardDeletedTime());
|
||||
assertNotNull(fileStatusService.getDossierStatus(dossierId)
|
||||
.get(1).getDeleted());
|
||||
|
||||
softDeletedFilesCleanupJob.execute(null);
|
||||
|
||||
Optional<FileModel> otherFileModel = fileStatusService.getDossierStatus(dossierId)
|
||||
.stream()
|
||||
.filter(fm -> fm.getId().equals(otherFileId))
|
||||
.findFirst();
|
||||
Assertions.assertTrue(otherFileModel.isPresent());
|
||||
|
||||
assertNotNull(otherFileModel.get().getHardDeletedTime());
|
||||
assertNotNull(otherFileModel.get().getDeleted());
|
||||
|
||||
|
||||
Optional<FileModel> fileModel = fileStatusService.getDossierStatus(dossierId)
|
||||
.stream()
|
||||
.filter(fm -> fm.getId().equals(fileId))
|
||||
.findFirst();
|
||||
Assertions.assertTrue(fileModel.isPresent());
|
||||
return fileModel.get();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testHardDeletedFilesCleanupJob() {
|
||||
|
||||
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
|
||||
|
||||
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
|
||||
String dossierId = dossier.getId();
|
||||
|
||||
var file = fileTesterAndProvider.testAndProvideFile(dossier);
|
||||
String fileId = file.getId();
|
||||
|
||||
OffsetDateTime withinRetryTime = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getHardDeleteCleanupRetryTime() / 24).plusHours(1);
|
||||
|
||||
executeDeletedFilesCleanupJobForGivenTime(withinRetryTime, dossierId, fileId);
|
||||
|
||||
Assertions.assertFalse(fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN));
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testHardDeletedFilesCleanupJobUnsuccessful() {
|
||||
|
||||
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
|
||||
|
||||
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
|
||||
String dossierId = dossier.getId();
|
||||
|
||||
var file = fileTesterAndProvider.testAndProvideFile(dossier);
|
||||
String fileId = file.getId();
|
||||
|
||||
OffsetDateTime tooMuchTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getHardDeleteCleanupRetryTime() / 24).minusHours(1);
|
||||
|
||||
executeDeletedFilesCleanupJobForGivenTime(tooMuchTimePassed, dossierId, fileId);
|
||||
|
||||
Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN));
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void executeDeletedFilesCleanupJobForGivenTime(OffsetDateTime offsetDateTime, String dossierId, String fileId) {
|
||||
|
||||
byte[] bytesToStore = {1, 2, 3, 4};
|
||||
fileManagementStorageService.storeObject(dossierId, fileId, FileType.ORIGIN, new ByteArrayInputStream(bytesToStore));
|
||||
|
||||
dossierService.hardDeleteDossier(dossierId, offsetDateTime);
|
||||
fileDeletionService.hardDeleteFiles(dossierId, List.of(fileId), offsetDateTime);
|
||||
|
||||
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
|
||||
|
||||
assertNotNull(dossierService.getDossierById(dossierId).getHardDeletedTime());
|
||||
assertNotNull(fileStatusService.getDossierStatus(dossierId)
|
||||
.get(0).getHardDeletedTime());
|
||||
Assertions.assertArrayEquals(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN), bytesToStore);
|
||||
|
||||
deletedFilesCleanupJob.execute(null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -22,7 +22,6 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mockito;
|
||||
import org.quartz.Scheduler;
|
||||
import org.springframework.amqp.core.AmqpAdmin;
|
||||
import org.springframework.amqp.rabbit.core.RabbitAdmin;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
|
||||
@ -311,7 +310,12 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
|
||||
TenantContext.setTenantId(TENANT_1);
|
||||
|
||||
ApplicationConfig appConfig = ApplicationConfig.builder().downloadCleanupDownloadFilesHours(8).downloadCleanupNotDownloadFilesHours(72).softDeleteCleanupTime(96).build();
|
||||
ApplicationConfig appConfig = ApplicationConfig.builder()
|
||||
.downloadCleanupDownloadFilesHours(8)
|
||||
.downloadCleanupNotDownloadFilesHours(72)
|
||||
.softDeleteCleanupTime(96)
|
||||
.hardDeleteCleanupRetryTime(72)
|
||||
.build();
|
||||
applicationConfigService.saveApplicationConfiguration(MagicConverter.convert(appConfig, ApplicationConfigurationEntity.class));
|
||||
|
||||
tokenService.setUser("manageradmin1@test.com", "secret");
|
||||
|
||||
@ -18,5 +18,7 @@ public class ApplicationConfig {
|
||||
private int downloadCleanupNotDownloadFilesHours;
|
||||
@Min(1)
|
||||
private int softDeleteCleanupTime;
|
||||
@Min(1)
|
||||
private int hardDeleteCleanupRetryTime;
|
||||
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user