From e0aa7838aa2db324927f88fd44b2c6e752ce6377 Mon Sep 17 00:00:00 2001 From: aoezyetimoglu Date: Tue, 12 Oct 2021 14:36:41 +0200 Subject: [PATCH 1/5] RED-2422: Port DeletedFilesCleanupService from filemanagement service --- .../scheduler/DeletedFilesCleanupService.java | 55 +++++++++++++++++++ .../FileManagementServiceSettings.java | 1 + 2 files changed, 56 insertions(+) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java new file mode 100644 index 000000000..059f77b87 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java @@ -0,0 +1,55 @@ +package com.iqser.red.service.peristence.v1.server.service.scheduler; + +import java.time.OffsetDateTime; +import java.util.List; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.peristence.v1.server.service.DossierService; +import com.iqser.red.service.peristence.v1.server.service.FileService; +import com.iqser.red.service.peristence.v1.server.service.FileStatusService; +import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +@Service +public class DeletedFilesCleanupService { + + private final DossierService dossierService; + private final FileStatusService fileStatusService; + private final FileService fileService; + private final FileManagementServiceSettings settings; + + + @Scheduled(fixedDelay = 300000, initialDelay = 300000) + public void timedDeletion() { + + var now = OffsetDateTime.now(); + List dossiers = dossierService.getAllDossiers(); + + for (DossierEntity dossierEntity : dossiers) { + if (dossierEntity.getSoftDeletedTime() != null && dossierEntity.getHardDeletedTime() == null) { + if (dossierEntity.getSoftDeletedTime().isBefore(now.minusHours(settings.getSoftDeleteCleanupTime()))) { + dossierService.hardDeleteDossier(dossierEntity.getId()); + log.info("Hard deleted dossier with dossier id {} ", dossierEntity.getId()); + } + } else { + var fileEntities = fileStatusService.getDossierStatus(dossierEntity.getId()); + for (FileEntity fileEntity : fileEntities) { + if (fileEntity.getHardDeletedTime() == null && fileEntity.getDeleted() != null && fileEntity.getDeleted() + .isBefore(now.minusHours(settings.getSoftDeleteCleanupTime()))) { + fileService.hardDeleteFile(dossierEntity.getId(), fileEntity.getId()); + log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), fileEntity.getId()); + } + } + } + } + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java index 6608279ed..36b550f39 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/settings/FileManagementServiceSettings.java @@ -17,5 +17,6 @@ public class FileManagementServiceSettings { private int downloadCleanupDownloadFilesHours = 8; private int downloadCleanupNotDownloadFilesHours = 72; + private int softDeleteCleanupTime = 96; } From 6122080fc94cc1479e52951e82f504f146fb29c3 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Wed, 13 Oct 2021 10:18:40 +0300 Subject: [PATCH 2/5] RED-2313 --- .../persistence/DossierTemplatePersistenceService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierTemplatePersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierTemplatePersistenceService.java index 063776106..a21bfec6d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierTemplatePersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierTemplatePersistenceService.java @@ -31,9 +31,10 @@ public class DossierTemplatePersistenceService { if (createOrUpdateDossierRequest.getDossierTemplateId() != null) { Optional dossierTemplate = dossierTemplateRepository.findById(createOrUpdateDossierRequest.getDossierTemplateId()); if (dossierTemplate.isPresent()) { + // order is important + BeanUtils.copyProperties(createOrUpdateDossierRequest, dossierTemplate.get()); dossierTemplate.get().setDateModified(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); dossierTemplate.get().setModifiedBy(createOrUpdateDossierRequest.getRequestingUser()); - BeanUtils.copyProperties(createOrUpdateDossierRequest, dossierTemplate.get()); return dossierTemplate.get(); } else { throw new NotFoundException(String.format(DOSSIER_TEMPLATE_NOT_FOUND_MESSAGE, createOrUpdateDossierRequest.getDossierTemplateId())); @@ -41,9 +42,10 @@ public class DossierTemplatePersistenceService { } else { DossierTemplateEntity dossierTemplate = new DossierTemplateEntity(); dossierTemplate.setId(UUID.randomUUID().toString()); + // order is important + BeanUtils.copyProperties(createOrUpdateDossierRequest, dossierTemplate); dossierTemplate.setDateAdded(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); dossierTemplate.setCreatedBy(createOrUpdateDossierRequest.getRequestingUser()); - BeanUtils.copyProperties(createOrUpdateDossierRequest, dossierTemplate); return dossierTemplateRepository.save(dossierTemplate); } From 6b707936ef4ee3fb4fbc2c9ae7ec8f313a6b1e8b Mon Sep 17 00:00:00 2001 From: aoezyetimoglu Date: Wed, 13 Oct 2021 12:27:20 +0200 Subject: [PATCH 3/5] RED-2422: Port DeletedFilesCleanupService from filemanagement service --- .../v1/server/service/scheduler/DeletedFilesCleanupService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java index 059f77b87..54c433bb9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java @@ -45,6 +45,7 @@ public class DeletedFilesCleanupService { if (fileEntity.getHardDeletedTime() == null && fileEntity.getDeleted() != null && fileEntity.getDeleted() .isBefore(now.minusHours(settings.getSoftDeleteCleanupTime()))) { fileService.hardDeleteFile(dossierEntity.getId(), fileEntity.getId()); + fileStatusService.setFileStatusHardDeleted(fileEntity.getId()); log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), fileEntity.getId()); } } From cee9ef35cab36356b4ee9435191d7d40a63e652c Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 14 Oct 2021 07:51:02 +0300 Subject: [PATCH 4/5] removed cleanup of viewed pages --- .../AnalysisFlagsCalculationService.java | 15 ------------- .../service/ManualRedactionService.java | 22 +++++++++---------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/AnalysisFlagsCalculationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/AnalysisFlagsCalculationService.java index cd6004534..311fbf9e0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/AnalysisFlagsCalculationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/AnalysisFlagsCalculationService.java @@ -1,6 +1,5 @@ package com.iqser.red.service.peristence.v1.server.service; -import com.google.common.collect.Sets; import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ViewedPageEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; @@ -12,9 +11,7 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.time.OffsetDateTime; -import java.util.ArrayList; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; @@ -30,11 +27,6 @@ public class AnalysisFlagsCalculationService { @Async public void calculateFlags(String dossierId, String fileId) { - this.calculateFlags(dossierId,fileId, Sets.newHashSet()); - } - - @Async - public void calculateFlags(String dossierId, String fileId, Set pagesToSkip) { long startTime = System.currentTimeMillis(); @@ -53,9 +45,6 @@ public class AnalysisFlagsCalculationService { boolean hasComments = false; - var viewedPagesToReset = new ArrayList(); - - for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { if (entry.isExcluded()) { continue; @@ -89,16 +78,12 @@ public class AnalysisFlagsCalculationService { var viewedPage = entry.getPositions().isEmpty() ? null : viewedPages.get(entry.getPositions().get(0).getPage()); if (lastChange != null && lastChange.getDateTime() != null && viewedPage != null && viewedPage.isBefore(lastChange.getDateTime())) { - viewedPagesToReset.add(entry.getPositions().get(0).getPage()); hasUpdates = true; } } - viewedPagesToReset.removeAll(pagesToSkip); - - viewedPagesPersistenceService.resetViewedPages(fileId, file.getCurrentReviewer(), viewedPagesToReset); fileStatusPersistenceService.updateFlags(fileId, hasRedactions, hasHints, hasImages, hasSuggestions, hasComments, hasUpdates); log.info("Flag Calculations for file: {} took: {}ms", fileId, System.currentTimeMillis() - startTime); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java index 0b75982be..ddee6d382 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ManualRedactionService.java @@ -88,7 +88,7 @@ public class ManualRedactionService { } if (!addRedactionRequest.getPositions().isEmpty()) { - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(addRedactionRequest.getPositions().get(0).getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } return ManualAddResponse.builder().annotationId(annotationId).commentId(commentId).build(); @@ -120,7 +120,7 @@ public class ManualRedactionService { } if (!removeRedactionRequest.isRemoveFromDictionary()) { - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(removeRedactionRequest.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } return ManualAddResponse.builder() @@ -151,7 +151,7 @@ public class ManualRedactionService { fileStatusPersistenceService.updateLastManualRedaction(fileId, now); } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(forceRedactionRequest.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); return ManualAddResponse.builder() .annotationId(forceRedactionRequest.getAnnotationId()) @@ -181,7 +181,7 @@ public class ManualRedactionService { fileStatusPersistenceService.updateLastManualRedaction(fileId, now); } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(legalBasisChangeRequest.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); return ManualAddResponse.builder() .annotationId(legalBasisChangeRequest.getAnnotationId()) @@ -286,7 +286,7 @@ public class ManualRedactionService { } if (!addRedaction.getPositions().isEmpty()) { - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(addRedaction.getPositions().get(0).getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } } @@ -307,7 +307,7 @@ public class ManualRedactionService { } else { fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now()); } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(removeRedaction.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -323,7 +323,7 @@ public class ManualRedactionService { } else { fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now()); } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(forceRedaction.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -338,7 +338,7 @@ public class ManualRedactionService { } else { fileStatusPersistenceService.updateLastManualRedaction(fileId, OffsetDateTime.now()); } - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(legalBasisChange.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -407,7 +407,7 @@ public class ManualRedactionService { boolean hasSuggestions = calculateHasSuggestions(fileId); fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions); - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(idRemoval.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } private String buildTypeId(RedactionLogEntry redactionLogEntry, DossierEntity dossier) { @@ -458,7 +458,7 @@ public class ManualRedactionService { recategorizationPersistenceService.updateStatus(fileId, annotationId, annotationStatus); boolean hasSuggestions = calculateHasSuggestions(fileId); fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions); - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(imageRecategorization.getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } @@ -484,7 +484,7 @@ public class ManualRedactionService { boolean hasSuggestions = calculateHasSuggestions(fileId); fileStatusPersistenceService.setUpdateLastManualRedactionAndHasSuggestions(fileId, OffsetDateTime.now(), hasSuggestions); if (!manualRedactionEntry.getPositions().isEmpty()) { - analysisFlagsCalculationService.calculateFlags(dossierId, fileId, Sets.newHashSet(manualRedactionEntry.getPositions().get(0).getPage())); + analysisFlagsCalculationService.calculateFlags(dossierId, fileId); } } From 1b77a6a9b90345b5e601fac06ebe7d6d7f1fc25e Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Thu, 14 Oct 2021 09:09:38 +0300 Subject: [PATCH 5/5] num analysis set now --- .../service/persistence/FileStatusPersistenceService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 4889ea3b6..6643e0c63 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -61,6 +61,7 @@ public class FileStatusPersistenceService { file.setLegalBasisVersion(legalBasisVersion); file.setAnalysisDuration(duration); file.setDossierDictionaryVersion(dossierDictionaryVersion); + file.setNumberOfAnalyses(file.getNumberOfAnalyses()+1); file.setAnalysisVersion(analysisVersion); file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); file.setLastProcessed(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));