From fac8a1ac85de5175a91f70a55e95ba1da03437e9 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Tue, 16 Apr 2024 18:25:07 +0300 Subject: [PATCH] RED-8966 Improved performance of soft and hard delete files and dossirs, and undelete - split transaction and vastly improved query performance --- .../impl/controller/DossierController.java | 3 +- .../service/DossierDeletionService.java | 59 ++++ .../service/DossierManagementService.java | 45 ++- .../service/FileDeletionService.java | 159 +++++++++++ .../v1/processor/service/FileService.java | 216 +------------- .../processor/service/FileStatusService.java | 56 +--- .../service/job/DeletedFilesCleanupJob.java | 7 +- .../FileStatusPersistenceService.java | 15 +- .../ViewedPagesPersistenceService.java | 5 +- .../AddRedactionPersistenceService.java | 16 +- .../CommentPersistenceService.java | 13 + .../ForceRedactionPersistenceService.java | 35 +-- .../LegalBasisChangePersistenceService.java | 34 +-- .../RecategorizationPersistenceService.java | 21 +- .../RemoveRedactionPersistenceService.java | 33 +-- .../ResizeRedactionPersistenceService.java | 34 +-- .../repository/CommentRepository.java | 30 +- .../repository/FileAttributesRepository.java | 4 + .../repository/FileRepository.java | 28 +- .../repository/ViewedPagesRepository.java | 5 + .../ForceRedactionRepository.java | 26 +- .../LegalBasisChangeRepository.java | 27 +- .../ManualRedactionRepository.java | 13 + .../RecategorizationRepository.java | 14 + .../RemoveRedactionRepository.java | 14 + .../ResizeRedactionRepository.java | 14 + .../integration/tests/SoftDeleteTest.java | 266 ++++++++++++++++++ .../AbstractPersistenceServerServiceTest.java | 8 + 28 files changed, 796 insertions(+), 404 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierDeletionService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SoftDeleteTest.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java index 29e1953ec..0fdfed37f 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DossierController.java @@ -367,7 +367,8 @@ public class DossierController implements DossierResource { throw new AccessDeniedException("Can not delete dossier that is owned by a different user"); } - dossierManagementService.delete(dossierId); + + dossierManagementService.softDeleteDossier(dossierId); auditPersistenceService.audit(AuditRequest.builder() .userId(KeycloakSecurity.getUserId()) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierDeletionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierDeletionService.java new file mode 100644 index 000000000..f1e91341e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierDeletionService.java @@ -0,0 +1,59 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.time.OffsetDateTime; +import java.util.List; + +import org.springframework.stereotype.Service; + +import jakarta.transaction.Transactional; +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class DossierDeletionService { + + private final FileDeletionService fileDeletionService; + private final DossierService dossierService; + + + @Transactional + public void undeleteDossier(String dossierId, List relevantFileIds, OffsetDateTime dossierSoftDeleteTime) { + + relevantFileIds.forEach(fileId -> { + fileDeletionService.undeleteFile(fileId, dossierSoftDeleteTime); + }); + + dossierService.undeleteDossier(dossierId); + } + + + @Transactional + public void softDeleteDossier(String dossierId, List relevantFileIds, OffsetDateTime softDeleteTime) { + + fileDeletionService.softDeleteFiles(relevantFileIds, softDeleteTime); + dossierService.softDeleteDossier(dossierId, softDeleteTime); + } + + + @Transactional + public void hardDeleteDossier(String dossierId, List relevantFileIds) { + + fileDeletionService.hardDeleteFiles(relevantFileIds); + dossierService.hardDeleteDossier(dossierId); + + } + + + public void hardDeleteFileDataAndIndexUpdates(String dossierId, List fileIds) { + + fileDeletionService.hardDeleteFileDataAndIndexUpdates(dossierId, fileIds); + + } + + + public void reindexUndeletedFiles(String dossierTemplateId, String dossierId, List relevantFileIds) { + + this.fileDeletionService.reindexUndeletedFiles(dossierTemplateId, dossierId, relevantFileIds); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java index e4c761c58..130317c51 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; import java.util.Set; @@ -35,6 +36,8 @@ public class DossierManagementService { private final FileStatusService fileStatusService; private final FileService fileService; private final IndexingService indexingService; + private final DossierDeletionService dossierDeletionService; + private final FileDeletionService fileDeletionService; public Set changesSince(JSONPrimitive since) { @@ -50,10 +53,9 @@ public class DossierManagementService { } - @Transactional - public void delete(String dossierId) { + public void softDeleteDossier(String dossierId) { - OffsetDateTime now = OffsetDateTime.now(); + OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); DossierEntity dossier = dossierService.getDossierById(dossierId); if (dossier.getSoftDeletedTime() != null /*|| dossier.getHardDeletedTime() != null*/) { @@ -61,14 +63,12 @@ public class DossierManagementService { } List fileStatuses = fileStatusService.getDossierStatus(dossierId); - fileStatuses.stream() - .filter(fileStatus -> fileStatus.getDeleted() == null) - .forEach(fileStatus -> { - fileService.softDeleteFile(dossierId, fileStatus.getId(), now); - fileStatusService.setFileStatusDeleted(fileStatus.getId(), now); - }); + var relevantFileIds = fileStatuses.stream() + .filter(fileStatus -> fileStatus.getDeleted() == null).map(FileModel::getId).toList(); - dossierService.softDeleteDossier(dossierId, now); + + dossierDeletionService.softDeleteDossier(dossierId, relevantFileIds, now); + fileDeletionService.reindexDeletedFiles(dossierId, relevantFileIds); } @@ -197,17 +197,13 @@ public class DossierManagementService { } - @Transactional public void hardDeleteDossiers(Set dossierIds) { for (String dossierId : dossierIds) { - DossierEntity dossier = dossierService.getDossierById(dossierId); - dossierService.hardDeleteDossier(dossier.getId()); - List fileStatuses = fileStatusService.getDossierStatus(dossierId); - fileStatuses.forEach(fileStatus -> { - fileService.hardDeleteFile(dossierId, fileStatus.getId()); - fileStatusService.setFileStatusHardDeleted(fileStatus.getId()); - }); + + List fileIds = fileStatusService.getDossierStatus(dossierId).stream().map(FileModel::getId).collect(Collectors.toList()); + dossierDeletionService.hardDeleteDossier(dossierId, fileIds); + dossierDeletionService.hardDeleteFileDataAndIndexUpdates(dossierId, fileIds); } } @@ -218,15 +214,12 @@ public class DossierManagementService { for (String dossierId : dossierIds) { var dossier = dossierService.getDossierById(dossierId); List fileStatuses = fileStatusService.getDossierStatus(dossierId); - fileStatuses.forEach(fileStatus -> { - if (fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted() - .isAfter(dossier.getSoftDeletedTime()))) { - fileService.undeleteFile(dossier.getDossierTemplateId(), dossierId, fileStatus.getId(), dossier.getSoftDeletedTime()); - fileStatusService.setFileStatusUndeleted(fileStatus.getId()); - } - }); + var relevantFileIds = fileStatuses.stream().filter(fileStatus -> fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted() + .isAfter(dossier.getSoftDeletedTime()))).map(FileModel::getId).collect(Collectors.toList()); + + dossierDeletionService.undeleteDossier(dossierId,relevantFileIds,dossier.getSoftDeletedTime()); + dossierDeletionService.reindexUndeletedFiles(dossier.getDossierTemplateId(), dossierId, relevantFileIds); - dossierService.undeleteDossier(dossierId); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java new file mode 100644 index 000000000..e8dfe18a4 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileDeletionService.java @@ -0,0 +1,159 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RecategorizationPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.search.v1.model.IndexMessageType; +import jakarta.transaction.Transactional; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class FileDeletionService { + + private final CommentPersistenceService commentPersistenceService; + private final ViewedPagesPersistenceService viewedPagesPersistenceService; + private final AddRedactionPersistenceService addRedactionPersistenceService; + private final ForceRedactionPersistenceService forceRedactionPersistenceService; + private final RemoveRedactionPersistenceService removeRedactionPersistenceService; + private final RecategorizationPersistenceService recategorizationPersistenceService; + private final LegalBasisChangePersistenceService legalBasisChangePersistenceService; + private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; + private final FileStatusPersistenceService fileStatusPersistenceService; + private final FileManagementStorageService fileManagementStorageService; + private final IndexingService indexingService; + + + public void undeleteFile(String fileId, OffsetDateTime softDeletedTime) { + + // revert comment deletion + commentPersistenceService.undeleteByFileId(fileId, softDeletedTime); + + // revert annotation deletion + addRedactionPersistenceService.undeleteByFileId(fileId, softDeletedTime); + forceRedactionPersistenceService.undeleteByFileId(fileId, softDeletedTime); + removeRedactionPersistenceService.undeleteByFileId(fileId, softDeletedTime); + recategorizationPersistenceService.undeleteByFileId(fileId, softDeletedTime); + legalBasisChangePersistenceService.undeleteByFileId(fileId, softDeletedTime); + resizeRedactionPersistenceService.undeleteByFileId(fileId, softDeletedTime); + + fileStatusPersistenceService.undelete(fileId); + } + + + @Transactional + public void softDeleteFiles(List fileIds, OffsetDateTime softDeleteTime) { + + // delete all annotations + addRedactionPersistenceService.softDeleteByFileIds(fileIds, softDeleteTime); + forceRedactionPersistenceService.softDeleteByFileIds(fileIds, softDeleteTime); + removeRedactionPersistenceService.softDeleteByFileIds(fileIds, softDeleteTime); + recategorizationPersistenceService.softDeleteByFileIds(fileIds, softDeleteTime); + legalBasisChangePersistenceService.softDeleteByFileIds(fileIds, softDeleteTime); + resizeRedactionPersistenceService.softDeleteByFileIds(fileIds, softDeleteTime); + + // delete all comments for all deleted annotations and files + commentPersistenceService.softDeleteCommentsForFiles(fileIds, softDeleteTime); + + // delete all viewed pages for all files + viewedPagesPersistenceService.deleteForFiles(fileIds); + + fileStatusPersistenceService.softDeleteFiles(fileIds, softDeleteTime); + + } + + + private void hardDeleteFileData(String dossierId, List fileIds) { + + fileIds.forEach(fileId -> { + Arrays.stream(FileType.values()) + .forEach(fileType -> { + try { + fileManagementStorageService.deleteObject(dossierId, fileId, fileType); + } catch (Exception e) { + log.warn("Failed to physically delete file: {} with type {}", fileId, fileType); + } + }); + }); + } + + + public void hardDeleteFileDataAndIndexUpdates(String dossierId, String fileId) { + + this.hardDeleteFileDataAndIndexUpdates(dossierId, List.of(fileId)); + + } + + + public void hardDeleteFileDataAndIndexUpdates(String dossierId, List fileIds) { + + hardDeleteFileData(dossierId, fileIds); + + fileIds.forEach(fileId -> { + indexingService.addToDeleteFromIndexQueue(dossierId, fileId, 2); + }); + + } + + + @Transactional + public void hardDeleteFile(String fileId) { + + this.hardDeleteFiles(List.of(fileId)); + } + + + @Transactional + public void hardDeleteFiles(List fileIds) { + + // delete all annotations + addRedactionPersistenceService.deleteByFileIds(fileIds); + forceRedactionPersistenceService.deleteByFileIds(fileIds); + removeRedactionPersistenceService.deleteByFileIds(fileIds); + recategorizationPersistenceService.deleteByFileIds(fileIds); + legalBasisChangePersistenceService.deleteByFileIds(fileIds); + resizeRedactionPersistenceService.deleteByFileIds(fileIds); + + // delete all comments for all deleted annotations and files + commentPersistenceService.deleteByFileIds(fileIds); + + // delete all viewed pages for all files + viewedPagesPersistenceService.deleteForFiles(fileIds); + + fileStatusPersistenceService.hardDeleteFiles(fileIds); + + } + + + public void reindexDeletedFiles(String dossierId, Collection relevantFileIds) { + + relevantFileIds.forEach(fileId -> { + indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2); + }); + } + + + public void reindexUndeletedFiles(String dossierTemplateId, String dossierId, Collection relevantFileIds) { + + relevantFileIds.forEach(fileId -> { + indexingService.addToIndexingQueue(IndexMessageType.UPDATE, dossierTemplateId, dossierId, fileId, 2); + }); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java index 131609776..1e19c6bd5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileService.java @@ -43,19 +43,11 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class FileService { - private final AddRedactionPersistenceService addRedactionPersistenceService; - private final CommentPersistenceService commentPersistenceService; - private final ForceRedactionPersistenceService forceRedactionPersistenceService; - private final RemoveRedactionPersistenceService removeRedactionPersistenceService; private final FileStatusService fileStatusService; - private final ViewedPagesPersistenceService viewedPagesPersistenceService; private final FileManagementStorageService fileManagementStorageService; private final DossierPersistenceService dossierPersistenceService; - private final RecategorizationPersistenceService recategorizationPersistenceService; - private final LegalBasisChangePersistenceService legalBasisChangePersistenceService; - private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; - private final IndexingService indexingService; private final DossierService dossierService; + private final FileDeletionService fileDeletionService; public JSONPrimitive upload(AddFileRequest request, boolean keepManualRedactions, long size) { @@ -108,59 +100,8 @@ public class FileService { } OffsetDateTime softDeleteTime = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); - softDeleteFile(dossierId, fileId, softDeleteTime); - fileStatusService.setFileStatusDeleted(fileId, softDeleteTime); - } - - - public void softDeleteFile(String dossierId, String fileId, OffsetDateTime softDeletedTime) { - - ManualChangesQueryOptions options = ManualChangesQueryOptions.allWithoutDeleted(); - - forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - forceRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false) - .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime)); - }); - - removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - removeRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false) - .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime)); - }); - - addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - addRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false) - .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime)); - }); - - recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(recatigorization -> { - recategorizationPersistenceService.softDelete(fileId, recatigorization.getId().getAnnotationId(), softDeletedTime); - commentPersistenceService.findCommentsByAnnotationId(fileId, recatigorization.getId().getAnnotationId(), false) - .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime)); - }); - - resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - resizeRedactionPersistenceService.softDelete(fileId, annotation.getId().getAnnotationId(), softDeletedTime); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), false) - .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime)); - }); - - legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(legalBasisChange -> { - legalBasisChangePersistenceService.softDelete(fileId, legalBasisChange.getId().getAnnotationId(), softDeletedTime); - commentPersistenceService.findCommentsByAnnotationId(fileId, legalBasisChange.getId().getAnnotationId(), false) - .forEach(comment -> commentPersistenceService.softDelete(comment.getId(), softDeletedTime)); - }); - - viewedPagesPersistenceService.deleteForFile(fileId); - indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2); + fileDeletionService.softDeleteFiles(List.of(fileId), softDeleteTime); + fileDeletionService.reindexDeletedFiles(dossierId, List.of(fileId)); } @@ -172,62 +113,8 @@ public class FileService { throw new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE); } - fileIds.forEach(fileId -> { - hardDeleteFile(dossierId, fileId); - fileStatusService.setFileStatusHardDeleted(fileId); - }); - } - - - public void hardDeleteFile(String dossierId, String fileId) { - - Arrays.stream(FileType.values()) - .forEach(fileType -> { - try { - fileManagementStorageService.deleteObject(dossierId, fileId, fileType); - } catch (Exception e) { - log.warn("Failed to physically delete file: {} with type {}", fileId, fileType); - } - }); - - ManualChangesQueryOptions options = ManualChangesQueryOptions.all(); - - forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - forceRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> commentPersistenceService.hardDelete(comment.getId())); - }); - - removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - removeRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> commentPersistenceService.hardDelete(comment.getId())); - }); - - addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - addRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> commentPersistenceService.hardDelete(comment.getId())); - }); - - recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(recatigorization -> { - recategorizationPersistenceService.hardDelete(fileId, recatigorization.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, recatigorization.getId().getAnnotationId(), true) - .forEach(comment -> commentPersistenceService.hardDelete(comment.getId())); - }); - - resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - resizeRedactionPersistenceService.hardDelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> commentPersistenceService.hardDelete(comment.getId())); - }); - indexingService.addToDeleteFromIndexQueue(dossierId, fileId, 2); - + fileDeletionService.hardDeleteFiles(fileIds); + fileDeletionService.hardDeleteFileDataAndIndexUpdates(dossierId, fileIds); } @@ -239,100 +126,11 @@ public class FileService { } for (String fileId : fileIds) { - FileModel fileStatus = fileStatusService.getStatus(fileId); OffsetDateTime softDeletedTime = fileStatus.getDeleted(); - - undeleteFile(dossier.getDossierTemplateId(), dossierId, fileId, softDeletedTime); - fileStatusService.setFileStatusUndeleted(fileId); + fileDeletionService.undeleteFile(fileId, softDeletedTime); } - } - - - public void undeleteFile(String dossierTemplateId, String dossierId, String fileId, OffsetDateTime softDeletedTime) { - - ManualChangesQueryOptions options = ManualChangesQueryOptions.all(); - forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) { - forceRedactionPersistenceService.undelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> { - if (comment.getSoftDeletedTime().equals(softDeletedTime) || comment.getSoftDeletedTime().isAfter(softDeletedTime)) { - commentPersistenceService.undelete(comment.getId()); - } - }); - } - }); - - removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) { - removeRedactionPersistenceService.undelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> { - if (comment.getSoftDeletedTime().equals(softDeletedTime) || comment.getSoftDeletedTime().isAfter(softDeletedTime)) { - commentPersistenceService.undelete(comment.getId()); - } - }); - } - }); - - addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - if (annotation != null && annotation.getSoftDeletedTime() != null && (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime() - .isAfter(softDeletedTime))) { - addRedactionPersistenceService.undelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> { - if (comment.getSoftDeletedTime().equals(softDeletedTime) || comment.getSoftDeletedTime().isAfter(softDeletedTime)) { - commentPersistenceService.undelete(comment.getId()); - } - }); - } - }); - - recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(recatigorization -> { - if (recatigorization.getSoftDeletedTime().equals(softDeletedTime) || recatigorization.getSoftDeletedTime().isAfter(softDeletedTime)) { - recategorizationPersistenceService.undelete(fileId, recatigorization.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, recatigorization.getId().getAnnotationId(), true) - .forEach(comment -> { - if (comment.getSoftDeletedTime().equals(softDeletedTime) || comment.getSoftDeletedTime().isAfter(softDeletedTime)) { - commentPersistenceService.undelete(comment.getId()); - } - }); - } - }); - - resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) { - resizeRedactionPersistenceService.undelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> { - if (comment.getSoftDeletedTime().equals(softDeletedTime) || comment.getSoftDeletedTime().isAfter(softDeletedTime)) { - commentPersistenceService.undelete(comment.getId()); - } - }); - } - }); - - legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, options) - .forEach(annotation -> { - if (annotation.getSoftDeletedTime().equals(softDeletedTime) || annotation.getSoftDeletedTime().isAfter(softDeletedTime)) { - legalBasisChangePersistenceService.undelete(fileId, annotation.getId().getAnnotationId()); - commentPersistenceService.findCommentsByAnnotationId(fileId, annotation.getId().getAnnotationId(), true) - .forEach(comment -> { - if (comment.getSoftDeletedTime().equals(softDeletedTime) || comment.getSoftDeletedTime().isAfter(softDeletedTime)) { - commentPersistenceService.undelete(comment.getId()); - } - }); - } - }); - - indexingService.addToIndexingQueue(IndexMessageType.UPDATE, dossierTemplateId, dossierId, fileId, 2); - + fileDeletionService.reindexUndeletedFiles(dossier.getDossierTemplateId(), dossierId, fileIds); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java index 4ad2d6cf3..7f750d5b1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java @@ -582,24 +582,6 @@ public class FileStatusService { } - public void setFileStatusDeleted(String fileId, OffsetDateTime softDeletedTime) { - - fileStatusPersistenceService.softDelete(fileId, softDeletedTime); - } - - - public void setFileStatusHardDeleted(String fileId) { - - fileStatusPersistenceService.hardDelete(fileId); - } - - - public void setFileStatusUndeleted(String fileId) { - - fileStatusPersistenceService.undelete(fileId); - } - - public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate) { fileStatusPersistenceService.updateFileModificationDate(fileId, fileManipulationDate); @@ -713,8 +695,7 @@ public class FileStatusService { } - @Transactional - public void deleteManualRedactions(String dossierId, String fileId) { + private void deleteManualRedactions(String dossierId, String fileId) { OffsetDateTime now = OffsetDateTime.now(); // remove everything related to redaction @@ -722,34 +703,13 @@ public class FileStatusService { fileManagementStorageService.deleteEntityLog(dossierId, fileId); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO); - // wipe comments - var comments = commentPersistenceService.findCommentsByFileID(fileId, false); - comments.forEach((key, value) -> value.forEach(comment -> commentPersistenceService.softDelete(comment.getId(), now))); - - // wipe force redactions - ManualChangesQueryOptions options = ManualChangesQueryOptions.allWithoutDeleted(); - var forceRedactions = forceRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options); - forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); - - // wipe add manual redactions - var addRedactions = addRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options); - addRedactions.forEach(f -> addRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); - - // wipe removeRedactions - var removeRedactions = removeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options); - removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); - - // wipe image recat - var imageRecategorizations = recategorizationPersistenceService.findEntriesByFileIdAndOptions(fileId, options); - imageRecategorizations.forEach(f -> recategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat - - // wipe resize redactions - var resizeRedactions = resizeRedactionPersistenceService.findEntriesByFileIdAndOptions(fileId, options); - resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); - - // wipe legal basis changes - var legalBasisChanges = legalBasisChangePersistenceService.findEntriesByFileIdAndOptions(fileId, options); - legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); + forceRedactionPersistenceService.softDeleteByFileIds(List.of(fileId), now); + addRedactionPersistenceService.softDeleteByFileIds(List.of(fileId), now); + removeRedactionPersistenceService.softDeleteByFileIds(List.of(fileId), now); + recategorizationPersistenceService.softDeleteByFileIds(List.of(fileId), now); + resizeRedactionPersistenceService.softDeleteByFileIds(List.of(fileId), now); + legalBasisChangePersistenceService.softDeleteByFileIds(List.of(fileId), now); + commentPersistenceService.softDeleteCommentsForFiles(List.of(fileId), now); fileStatusPersistenceService.updateHasComments(fileId, false); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java index 01ea1e5c7..3c00b8a10 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/job/DeletedFilesCleanupJob.java @@ -3,6 +3,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.job; import java.time.OffsetDateTime; import java.util.List; +import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.springframework.stereotype.Service; @@ -27,6 +28,7 @@ public class DeletedFilesCleanupJob implements Job { private final DossierService dossierService; private final FileStatusService fileStatusService; private final FileService fileService; + private final FileDeletionService fileDeletionService; private final ApplicationConfigService applicationConfigService; private final TenantProvider tenantProvider; @@ -58,8 +60,9 @@ public class DeletedFilesCleanupJob implements Job { for (var file : files) { if (file.getHardDeletedTime() == null && file.getDeleted() != null && file.getDeleted() .isBefore(now.minusHours(applicationConfigurationEntity.getSoftDeleteCleanupTime()))) { - fileService.hardDeleteFile(dossierEntity.getId(), file.getId()); - fileStatusService.setFileStatusHardDeleted(file.getId()); + + fileDeletionService.hardDeleteFile(file.getId()); + fileDeletionService.hardDeleteFileDataAndIndexUpdates(dossierEntity.getId(),file.getId()); log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), file.getId()); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 69abd670b..d07367561 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -403,7 +403,7 @@ public class FileStatusPersistenceService { var now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); - int countUpdate = fileRepository.setHardDelete(fileId, ProcessingStatus.PROCESSED, now, now, now); + int countUpdate = fileRepository.hardDeleteFiles(List.of(fileId), ProcessingStatus.PROCESSED, now); if (countUpdate == 0) { throw new NotFoundException(String.format("File with ID \"%s\" not found!", fileId)); } @@ -420,6 +420,19 @@ public class FileStatusPersistenceService { } } + public void softDeleteFiles(List fileIds, OffsetDateTime softDeletedTime) { + fileRepository.softDeleteFiles(fileIds, ProcessingStatus.PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), softDeletedTime); + } + + + + public void hardDeleteFiles(List fileIds) { + + var now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); + fileRepository.hardDeleteFiles(fileIds, ProcessingStatus.PROCESSED, now); + fileAttributesRepository.deleteByFileIds(fileIds); + } + @Transactional public void undelete(String fileId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ViewedPagesPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ViewedPagesPersistenceService.java index 5a52208ca..a5ae0e9bb 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ViewedPagesPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/ViewedPagesPersistenceService.java @@ -55,10 +55,9 @@ public class ViewedPagesPersistenceService { } - @Transactional - public void resetViewedPages(String fileId, String currentReviewer, List viewedPagesToReset) { + public void deleteForFiles(List fileIds) { + viewedPagesRepository.deleteByFileIds(fileIds); - viewedPagesRepository.deleteSeenPages(fileId, currentReviewer, viewedPagesToReset); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java index a2b6c93bf..a210ed259 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/AddRedactionPersistenceService.java @@ -87,9 +87,15 @@ public class AddRedactionPersistenceService { } - public List findAllAddRedactions() { + public int softDeleteByFileIds(List fileIds, OffsetDateTime softDeletionTime) { - return manualRedactionRepository.findAll(); + return manualRedactionRepository.softDeleteByFileIds(fileIds, softDeletionTime); + } + + + public int deleteByFileIds(List fileIds) { + + return manualRedactionRepository.deleteByFileIds(fileIds); } @@ -140,4 +146,10 @@ public class AddRedactionPersistenceService { manualRedactionRepository.markAsProcessed(new AnnotationEntityId(e.getAnnotationId(), e.getFileId()), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } + + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { + + manualRedactionRepository.undeleteByFileId(fileId, deletionTime); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java index c6d45b56a..c905d0db3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/CommentPersistenceService.java @@ -55,6 +55,19 @@ public class CommentPersistenceService { return commentRepository.existsByFileIdAndSoftDeletedTimeIsNull(fileId); } + public void softDeleteCommentsForFiles(List fileId, OffsetDateTime softDeletedTime) { + commentRepository.softDeleteCommentsByFilesAndAnnotationSoftDeletedTime(fileId, softDeletedTime); + } + + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { + + commentRepository.undeleteByFileId(fileId, deletionTime); + } + + public void deleteByFileIds(List fileIds) { + commentRepository.deleteCommentsByFiles(fileIds); + } + @Transactional public void hardDelete(long commentId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java index 5b756439c..ffdb7a74e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ForceRedactionPersistenceService.java @@ -43,6 +43,16 @@ public class ForceRedactionPersistenceService { return forceRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed()); } + public int softDeleteByFileIds(List fileIds, OffsetDateTime softDeletionTime) { + + return forceRedactionRepository.softDeleteByFileIds(fileIds, softDeletionTime); + } + + public int deleteByFileIds(List fileIds) { + + return forceRedactionRepository.deleteByFileIds(fileIds); + } + @Transactional public void hardDelete(String fileId, String annotationId) { @@ -64,30 +74,15 @@ public class ForceRedactionPersistenceService { forceRedactionRepository.updateSoftDelete(new AnnotationEntityId(annotationId, fileId), null); } - - public ManualForceRedactionEntity findForceRedaction(String fileId, String annotationId) { - - return forceRedactionRepository.findByIdAndNotSoftDeleted(new AnnotationEntityId(annotationId, fileId)) - .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); - } - - - public Set findForceRedactions(String fileId, boolean includeDeletions, boolean includeDictChanges) { - - return new HashSet<>(forceRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions)); - } - - - public Set findUnprocessedForceRedactions(String fileId, boolean includeDictChanges) { - - return new HashSet<>(forceRedactionRepository.findByFileIdAndUnprocessed(fileId)); - } - - @Transactional public void markAsProcessed(String annotationId, String fileId) { forceRedactionRepository.markAsProcessed(new AnnotationEntityId(annotationId, fileId), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { + + forceRedactionRepository.undeleteByFileId(fileId, deletionTime); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java index e88c1e1ca..4d78698dd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/LegalBasisChangePersistenceService.java @@ -74,25 +74,6 @@ public class LegalBasisChangePersistenceService { } - public ManualLegalBasisChangeEntity findLegalBasisChange(String fileId, String annotationId) { - - return legalBasisChangeRepository.findByIdAndNotSoftDeleted(new AnnotationEntityId(annotationId, fileId)) - .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); - } - - - public Set findLegalBasisChanges(String fileId, boolean includeDeletions, boolean includeDictChanges) { - - return new HashSet<>(legalBasisChangeRepository.findByFileIdIncludeDeletions(fileId, includeDeletions)); - } - - - public Set findUnprocessedLegalBasisChanges(String fileId) { - - return new HashSet<>(legalBasisChangeRepository.findUnprocessedByFileId(fileId)); - } - - public void markAsProcessed(String annotationId, String fileId) { legalBasisChangeRepository.markAsProcessed(new AnnotationEntityId(annotationId, fileId), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); @@ -104,4 +85,19 @@ public class LegalBasisChangePersistenceService { return legalBasisChangeRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed()); } + public int softDeleteByFileIds(List fileIds, OffsetDateTime softDeletionTime) { + + return legalBasisChangeRepository.softDeleteByFileIds(fileIds, softDeletionTime); + } + + public int deleteByFileIds(List fileIds) { + + return legalBasisChangeRepository.deleteByFileIds(fileIds); + } + + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { + + legalBasisChangeRepository.undeleteByFileId(fileId, deletionTime); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RecategorizationPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RecategorizationPersistenceService.java index e53b68948..ea7f3908f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RecategorizationPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RecategorizationPersistenceService.java @@ -92,24 +92,20 @@ public class RecategorizationPersistenceService { .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); } - - public List findRecategorizations(String fileId, boolean includeDeletions, boolean includeDictChanges) { - - return recategorizationRepository.findByFileIdIncludeDeletions(fileId, includeDeletions); - } - - public List findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) { return recategorizationRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges()); } + public int softDeleteByFileIds(List fileIds, OffsetDateTime softDeletionTime) { - public List findUnprocessedRecategorizations(String fileId, boolean includeDictChanges) { - - return recategorizationRepository.findUnprocessedByFileId(fileId); + return recategorizationRepository.softDeleteByFileIds(fileIds, softDeletionTime); } + public int deleteByFileIds(List fileIds) { + + return recategorizationRepository.deleteByFileIds(fileIds); + } @Transactional public void markAsProcessed(String annotationId, String fileId) { @@ -117,4 +113,9 @@ public class RecategorizationPersistenceService { recategorizationRepository.markAsProcessed(new AnnotationEntityId(annotationId, fileId), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); } + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { + + recategorizationRepository.undeleteByFileId(fileId, deletionTime); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java index dc213dd2e..a8e9f2c4f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/RemoveRedactionPersistenceService.java @@ -47,30 +47,25 @@ public class RemoveRedactionPersistenceService { .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); } - - public Set findRemoveRedactions(String fileId, boolean includeDeletions, boolean includeDictChanges) { - - return new HashSet<>(removeRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions)); - } - - - public Set findUnprocessedRemoveRedactions(String fileId, boolean includeDictChanges) { - - return new HashSet<>(removeRedactionRepository.findByFileIdAndUnprocessed(fileId)); - } - - - public List findAllRemoveRedactions() { - - return removeRedactionRepository.findAll(); - } - - public List findEntriesByFileIdAndOptions(String fileId, ManualChangesQueryOptions options) { return removeRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges()); } + public int softDeleteByFileIds(List fileIds, OffsetDateTime softDeletionTime) { + + return removeRedactionRepository.softDeleteByFileIds(fileIds, softDeletionTime); + } + + public int deleteByFileIds(List fileIds) { + + return removeRedactionRepository.deleteByFileIds(fileIds); + } + + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { + + removeRedactionRepository.undeleteByFileId(fileId, deletionTime); + } @Transactional public void hardDelete(String fileId, String annotationId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java index b81031a1e..e93d735ef 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/annotations/ResizeRedactionPersistenceService.java @@ -81,31 +81,12 @@ public class ResizeRedactionPersistenceService { } - public ManualResizeRedactionEntity findResizeRedaction(String fileId, String annotationId) { - - return resizeRedactionRepository.findByIdAndNotSoftDeleted(new AnnotationEntityId(annotationId, fileId)) - .orElseThrow(() -> new NotFoundException("Unknown file/annotation combination: " + fileId + "/" + annotationId)); - } - - public Optional findResizeRedactionById(String fileId, String annotationId) { return resizeRedactionRepository.findById(new AnnotationEntityId(annotationId, fileId)); } - public List findResizeRedactions(String fileId, boolean includeDeletions, boolean includeDictChanges) { - - return resizeRedactionRepository.findByFileIdIncludeDeletions(fileId, includeDeletions); - } - - - public List findUnprocessedResizeRedactions(String fileId) { - - return resizeRedactionRepository.findUnprocessedByFileId(fileId); - } - - public List findByValue(String value) { return resizeRedactionRepository.findByValue(value); @@ -123,4 +104,19 @@ public class ResizeRedactionPersistenceService { return resizeRedactionRepository.findByFileIdAndOptions(fileId, options.isIncludeDeletions(), options.isIncludeOnlyUnprocessed(), options.isIncludeDictChanges()); } + public int softDeleteByFileIds(List fileIds, OffsetDateTime softDeletionTime) { + + return resizeRedactionRepository.softDeleteByFileIds(fileIds, softDeletionTime); + } + + public int deleteByFileIds(List fileIds) { + + return resizeRedactionRepository.deleteByFileIds(fileIds); + } + + public void undeleteByFileId(String fileId, OffsetDateTime deletionTime) { + + resizeRedactionRepository.undeleteByFileId(fileId, deletionTime); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/CommentRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/CommentRepository.java index 689d0a5ac..75830cbec 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/CommentRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/CommentRepository.java @@ -27,17 +27,43 @@ public interface CommentRepository extends JpaRepository { @Query("select e from CommentEntity e where e.fileId = :fileId and (:includeDeletions = true or e.softDeletedTime is null)") List findByFileId(@Param("fileId") String fileId, @Param("includeDeletions") boolean includeDeletions); - boolean existsByFileIdAndSoftDeletedTimeIsNull(String fileId); - @Modifying @Query("update CommentEntity c set c.softDeletedTime = :softDeleteTime where c.id = :id") int updateSoftDelete(@Param("id") long id, @Param("softDeleteTime") OffsetDateTime softDeleteTime); + @Modifying + @Query("update CommentEntity c set c.softDeletedTime = :softDeleteTime where c.fileId in (:fileIds) and " + + "(exists (select a from ManualForceRedactionEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime = :softDeleteTime) or "+ + "exists (select a from IdRemovalEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime = :softDeleteTime) or "+ + "exists (select a from ManualRedactionEntryEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime = :softDeleteTime) or "+ + "exists (select a from ManualRecategorizationEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime = :softDeleteTime) or "+ + "exists (select a from ManualResizeRedactionEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime = :softDeleteTime) or "+ + "exists (select a from ManualLegalBasisChangeEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime = :softDeleteTime)) ") + void softDeleteCommentsByFilesAndAnnotationSoftDeletedTime(@Param("fileIds") List fileIds, @Param("softDeleteTime") OffsetDateTime softDeleteTime); + + @Modifying + @Query("delete from CommentEntity c where c.fileId in (:fileIds)") + void deleteCommentsByFiles(@Param("fileIds") List fileIds); + + + @Modifying + @Query("update CommentEntity c set c.softDeletedTime = null where c.fileId = :fileId and " + + "(exists (select a from ManualForceRedactionEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime >= :deletionTime) or "+ + "exists (select a from IdRemovalEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime >= :deletionTime) or "+ + "exists (select a from ManualRedactionEntryEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime >= :deletionTime) or "+ + "exists (select a from ManualRecategorizationEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime >= :deletionTime) or "+ + "exists (select a from ManualResizeRedactionEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime >= :deletionTime) or "+ + "exists (select a from ManualLegalBasisChangeEntity a where a.id.annotationId = c.annotationId and a.softDeletedTime >= :deletionTime)) ") + void undeleteByFileId(@Param("fileId") String fileId,@Param("deletionTime") OffsetDateTime deletionTime); + @Modifying @Query("update CommentEntity c set c.annotationId = :newAnnotationId where c.annotationId = :oldAnnotationId and c.fileId = :fileId") int saasMigrationUpdateAnnotationIds(@Param("fileId") String fileId, @Param("oldAnnotationId") String oldAnnotationId, @Param("newAnnotationId") String newAnnotationId); + + + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java index 88cb7ea70..8c73c91ad 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java @@ -20,6 +20,10 @@ public interface FileAttributesRepository extends JpaRepository fileIds); + @Query("SELECT f FROM FileAttributeEntity f where f.fileAttributeId.fileId = :fileId") List findByFileId(@Param("fileId") String fileId); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 5381f3fed..775564d99 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -168,20 +168,12 @@ public interface FileRepository extends JpaRepository { @Param("lastUpdated") OffsetDateTime lastUpdated, @Param("softDeletedTime") OffsetDateTime softDeletedTime); - @Modifying - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," - + "f.hardDeletedTime = :hardDeletedTime, " - + "f.deleted = case " - + " when f.deleted is null then :deleted " - + " when f.deleted is not null then f.deleted " - + "end " - + "where f.id = :fileId") - int setHardDelete(@Param("fileId") String fileId, + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id in (:fileIds)") + int softDeleteFiles(@Param("fileIds") List fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("lastUpdated") OffsetDateTime lastUpdated, - @Param("hardDeletedTime") OffsetDateTime hardDeletedTime, - @Param("deleted") OffsetDateTime deleted); + @Param("softDeletedTime") OffsetDateTime softDeletedTime); @Modifying @@ -379,6 +371,20 @@ public interface FileRepository extends JpaRepository { """) List getFileIdentifiersWhereAnalysisFlagCalculationIsRequired(); + + @Modifying + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :deletionTime," + + "f.hardDeletedTime = :deletionTime, " + + "f.deleted = case " + + " when f.deleted is null then :deletionTime " + + " when f.deleted is not null then f.deleted " + + "end " + + "where f.id in (:fileIds)") + int hardDeleteFiles(@Param("fileIds") List fileIds, + @Param("processingStatus") ProcessingStatus processingStatus, + @Param("deletionTime") OffsetDateTime deletionTime); + + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ViewedPagesRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ViewedPagesRepository.java index 3aa019fa0..96f1bb250 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ViewedPagesRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/ViewedPagesRepository.java @@ -21,4 +21,9 @@ public interface ViewedPagesRepository extends JpaRepository viewedPagesToReset); + + @Modifying + @Query("DELETE FROM ViewedPageEntity e where e.id.fileId in (:fileIds)") + void deleteByFileIds(@Param("fileIds")List fileIds); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ForceRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ForceRedactionRepository.java index 3b5ba3b97..9161b965d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ForceRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ForceRedactionRepository.java @@ -37,14 +37,28 @@ public interface ForceRedactionRepository extends JpaRepository findByFileIdAndOptions(@Param("fileId") String fileId, @Param("includeDeletions") boolean includeDeletions, @Param("unprocessed") boolean unprocessed); + + @Modifying + @Query("update ManualForceRedactionEntity m set m.softDeletedTime = :softDeletedTime where m.id.fileId in (:fileIds) and m.softDeletedTime is null") + int softDeleteByFileIds(@Param("fileIds") List fileIds, @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("update ManualForceRedactionEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime") + int undeleteByFileId(@Param("fileId") String fileId, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("delete ManualForceRedactionEntity m where m.id.fileId in (:fileIds)") + int deleteByFileIds(@Param("fileIds") List fileIds); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/LegalBasisChangeRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/LegalBasisChangeRepository.java index 2d8a0017b..af39ad66e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/LegalBasisChangeRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/LegalBasisChangeRepository.java @@ -38,14 +38,29 @@ public interface LegalBasisChangeRepository extends JpaRepository findByFileIdAndOptions(@Param("fileId") String fileId, @Param("includeDeletions") boolean includeDeletions, @Param("unprocessed") boolean unprocessed); + + @Modifying + @Query("update ManualLegalBasisChangeEntity m set m.softDeletedTime = :softDeletedTime where m.id.fileId in (:fileIds) and m.softDeletedTime is null") + int softDeleteByFileIds(@Param("fileIds") List fileIds, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("update ManualLegalBasisChangeEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime") + int undeleteByFileId(@Param("fileId") String fileId, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("delete ManualLegalBasisChangeEntity m where m.id.fileId in (:fileIds)") + int deleteByFileIds(@Param("fileIds") List fileIds); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java index 517da7d84..63f166de7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ManualRedactionRepository.java @@ -51,6 +51,19 @@ public interface ManualRedactionRepository extends JpaRepository fileIds, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("update ManualRedactionEntryEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime") + int undeleteByFileId(@Param("fileId") String fileId, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("delete ManualRedactionEntryEntity m where m.id.fileId in (:fileIds)") + int deleteByFileIds(@Param("fileIds") List fileIds); @Modifying @Query("update ManualRedactionEntryEntity m set m.processedDate = :processedDate where m.id = :annotationEntityId") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RecategorizationRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RecategorizationRepository.java index 1927f8a1b..334453178 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RecategorizationRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RecategorizationRepository.java @@ -49,4 +49,18 @@ public interface RecategorizationRepository extends JpaRepository fileIds, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("update ManualRecategorizationEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime") + int undeleteByFileId(@Param("fileId") String fileId, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("delete ManualRecategorizationEntity m where m.id.fileId in (:fileIds)") + int deleteByFileIds(@Param("fileIds") List fileIds); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java index d23cee427..4ece4c92d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/RemoveRedactionRepository.java @@ -49,4 +49,18 @@ public interface RemoveRedactionRepository extends JpaRepository fileIds, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("update IdRemovalEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime") + int undeleteByFileId(@Param("fileId") String fileId, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("delete IdRemovalEntity m where m.id.fileId in (:fileIds)") + int deleteByFileIds(@Param("fileIds") List fileIds); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ResizeRedactionRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ResizeRedactionRepository.java index 88554ee76..7b0394fa6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ResizeRedactionRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/annotationentity/ResizeRedactionRepository.java @@ -57,4 +57,18 @@ public interface ResizeRedactionRepository extends JpaRepository fileIds, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("update ManualResizeRedactionEntity m set m.softDeletedTime = null where m.id.fileId = :fileId and m.softDeletedTime >= :softDeletedTime") + int undeleteByFileId(@Param("fileId") String fileId, + @Param("softDeletedTime") OffsetDateTime softDeletedTime); + + @Modifying + @Query("delete ManualResizeRedactionEntity m where m.id.fileId in (:fileIds)") + int deleteByFileIds(@Param("fileIds") List fileIds); + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SoftDeleteTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SoftDeleteTest.java new file mode 100644 index 000000000..a1e69b19a --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/SoftDeleteTest.java @@ -0,0 +1,266 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.AnnotationEntityId; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.IdRemovalEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualForceRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualLegalBasisChangeEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRecategorizationEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualRedactionEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.ManualResizeRedactionEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.RectangleEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Test; + +@Slf4j +public class SoftDeleteTest extends AbstractPersistenceServerServiceTest { + + @Autowired + private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider; + + @Autowired + private DossierTesterAndProvider dossierTesterAndProvider; + + @Autowired + private FileTesterAndProvider fileTesterAndProvider; + + @Autowired + private DossierManagementService dossierManagementService; + + + @Test + public void testSoftDeleteDossier() { + + var template = dossierTemplateTesterAndProvider.provideTestTemplate("template1"); + var dossier = prepareDossier(template, "dossier1"); + long startSoftDelete = System.currentTimeMillis(); + dossierManagementService.softDeleteDossier(dossier.getDossierId()); + log.warn("Soft delete time: {}ms", System.currentTimeMillis() - startSoftDelete); + + var deletedDossier = dossierRepository.findById(dossier.getId()); + var allComments = commentRepository.findAll(); + var allManualRedactions = manualRedactionRepository.findAll(); + var allIdRemovals = removeRedactionRepository.findAll(); + var allRecategorizations = recategorizationRepository.findAll(); + var allLegalBasis = legalBasisChangeRepository.findAll(); + var allForceRedactions = forceRedactionRepository.findAll(); + var allResizes = resizeRedactionRepository.findAll(); + + assertThat(allComments.stream().map(CommentEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsExactly(deletedDossier.get().getSoftDeletedTime()); + assertThat(allManualRedactions.stream().map(ManualRedactionEntryEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsExactly(deletedDossier.get() + .getSoftDeletedTime()); + assertThat(allIdRemovals.stream().map(IdRemovalEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsExactly(deletedDossier.get().getSoftDeletedTime()); + assertThat(allRecategorizations.stream().map(ManualRecategorizationEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsExactly(deletedDossier.get() + .getSoftDeletedTime()); + assertThat(allLegalBasis.stream().map(ManualLegalBasisChangeEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsExactly(deletedDossier.get() + .getSoftDeletedTime()); + assertThat(allForceRedactions.stream().map(ManualForceRedactionEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsExactly(deletedDossier.get() + .getSoftDeletedTime()); + assertThat(allResizes.stream().map(ManualResizeRedactionEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsExactly(deletedDossier.get().getSoftDeletedTime()); + + long startUndeleteTime = System.currentTimeMillis(); + dossierManagementService.undeleteDossiers(Set.of(dossier.getDossierId())); + log.warn("Undo soft delete time: {}ms", System.currentTimeMillis() - startUndeleteTime); + + allComments = commentRepository.findAll(); + allManualRedactions = manualRedactionRepository.findAll(); + allIdRemovals = removeRedactionRepository.findAll(); + allRecategorizations = recategorizationRepository.findAll(); + allLegalBasis = legalBasisChangeRepository.findAll(); + allForceRedactions = forceRedactionRepository.findAll(); + allResizes = resizeRedactionRepository.findAll(); + + assertThat(allComments.stream().map(CommentEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsOnlyNulls(); + assertThat(allManualRedactions.stream().map(ManualRedactionEntryEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsOnlyNulls(); + assertThat(allIdRemovals.stream().map(IdRemovalEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsOnlyNulls(); + assertThat(allRecategorizations.stream().map(ManualRecategorizationEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsOnlyNulls(); + assertThat(allLegalBasis.stream().map(ManualLegalBasisChangeEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsOnlyNulls(); + assertThat(allForceRedactions.stream().map(ManualForceRedactionEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsOnlyNulls(); + assertThat(allResizes.stream().map(ManualResizeRedactionEntity::getSoftDeletedTime).collect(Collectors.toSet())).containsOnlyNulls(); + + long startHardDeleteTime = System.currentTimeMillis(); + dossierManagementService.hardDeleteDossiers(Set.of(dossier.getDossierId())); + log.warn("Hard Delete Time: {}ms", System.currentTimeMillis() - startHardDeleteTime); + + allComments = commentRepository.findAll(); + allManualRedactions = manualRedactionRepository.findAll(); + allIdRemovals = removeRedactionRepository.findAll(); + allRecategorizations = recategorizationRepository.findAll(); + allLegalBasis = legalBasisChangeRepository.findAll(); + allForceRedactions = forceRedactionRepository.findAll(); + allResizes = resizeRedactionRepository.findAll(); + + assertThat(allComments).isEmpty(); + assertThat(allManualRedactions).isEmpty(); + assertThat(allIdRemovals).isEmpty(); + assertThat(allRecategorizations).isEmpty(); + assertThat(allLegalBasis).isEmpty(); + assertThat(allForceRedactions).isEmpty(); + assertThat(allResizes).isEmpty(); + } + + + public Dossier prepareDossier(DossierTemplateModel template, String name) { + + var dossier = dossierTesterAndProvider.provideTestDossier(template, name); + + var manualRedactions = new ArrayList(); + var idRemovals = new ArrayList(); + var forceRedactions = new ArrayList(); + var legalBasisChanges = new ArrayList(); + var recategorizations = new ArrayList(); + var resizes = new ArrayList(); + var comments = new ArrayList(); + for (int i = 0; i < 20; i++) { + var file = fileTesterAndProvider.testAndProvideFile(dossier, "file" + i); + + for (int j = 0; j < 10; j++) { + + var manualRedaction = getManualRedactionEntryEntity(i, j, file); + var idRemoval = getIdRemovalEntity(i, j, file); + var forceRedaction = getManualForceRedactionEntity(i, j, file); + var legalBasisChange = getManualLegalBasisChangeEntity(i, j, file); + var recategorization = getManualRecategorizationEntity(i, j, file); + var resize = getManualResizeRedactionEntity(i, j, file); + manualRedactions.add(manualRedaction); + idRemovals.add(idRemoval); + forceRedactions.add(forceRedaction); + legalBasisChanges.add(legalBasisChange); + recategorizations.add(recategorization); + resizes.add(resize); + + for (int k = 0; k < 10; k++) { + var comment = new CommentEntity(); + comment.setAnnotationId(manualRedaction.getId().getAnnotationId()); + comment.setFileId(file.getFileId()); + comment.setText("test"); + comment.setUser("test"); + comments.add(comment); + } + + } + + } + + commentRepository.saveAll(comments); + manualRedactionRepository.saveAll(manualRedactions); + removeRedactionRepository.saveAll(idRemovals); + forceRedactionRepository.saveAll(forceRedactions); + legalBasisChangeRepository.saveAll(legalBasisChanges); + recategorizationRepository.saveAll(recategorizations); + resizeRedactionRepository.saveAll(resizes); + + return dossier; + } + + + private static @NotNull ManualResizeRedactionEntity getManualResizeRedactionEntity(int i, int j, FileStatus file) { + + var entity = new ManualResizeRedactionEntity(); + entity.setId(new AnnotationEntityId("ManualResizeRedactionEntity:" + i + ":" + j, file.getFileId())); + entity.setPositions(List.of(new RectangleEntity(1, 1, 1, 1, 1))); + entity.setUser("test"); + entity.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setUpdateDictionary(false); + entity.setTextAfter("test"); + entity.setTextBefore("test"); + entity.setValue("test"); + return entity; + } + + + private static @NotNull ManualRecategorizationEntity getManualRecategorizationEntity(int i, int j, FileStatus file) { + + var entity = new ManualRecategorizationEntity(); + entity.setId(new AnnotationEntityId("ManualRecategorizationEntity:" + i + ":" + j, file.getFileId())); + entity.setUser("test"); + entity.setLegalBasis("test"); + entity.setTypeId("123"); + entity.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setSection("test"); + entity.setValue("test"); + return entity; + } + + + private static @NotNull ManualLegalBasisChangeEntity getManualLegalBasisChangeEntity(int i, int j, FileStatus file) { + + var entity = new ManualLegalBasisChangeEntity(); + entity.setId(new AnnotationEntityId("ManualLegalBasisChangeEntity:" + i + ":" + j, file.getFileId())); + entity.setUser("test"); + entity.setLegalBasis("test"); + entity.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setSection("test"); + entity.setValue("test"); + return entity; + } + + + private static @NotNull IdRemovalEntity getIdRemovalEntity(int i, int j, FileStatus file) { + + var entity = new IdRemovalEntity(); + entity.setId(new AnnotationEntityId("IdRemovalEntity:" + i + ":" + j, file.getFileId())); + entity.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setUser("test"); + entity.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + return entity; + } + + + private static @NotNull ManualForceRedactionEntity getManualForceRedactionEntity(int i, int j, FileStatus file) { + + var entity = new ManualForceRedactionEntity(); + entity.setId(new AnnotationEntityId("IdRemovalEntity:" + i + ":" + j, file.getFileId())); + entity.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setUser("test"); + entity.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setPage(1); + return entity; + } + + + private static @NotNull ManualRedactionEntryEntity getManualRedactionEntryEntity(int i, int j, FileStatus file) { + + var entity = new ManualRedactionEntryEntity(); + entity.setId(new AnnotationEntityId("ManualRedactionEntryEntity:" + i + ":" + j, file.getFileId())); + entity.setPositions(List.of(new RectangleEntity(1, 1, 1, 1, 1))); + entity.setUser("test"); + entity.setLegalBasis("test"); + entity.setDictionaryEntryType(DictionaryEntryType.ENTRY); + entity.setReason("test"); + entity.setRequestDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setProcessedDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + entity.setRectangle(true); + entity.setSection("test"); + entity.setSourceId("test"); + entity.setTextAfter("test"); + entity.setTextBefore("test"); + entity.setValue("test"); + return entity; + } + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index d5bcde7b5..bd634866d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -11,6 +11,8 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.CommentRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ResizeRedactionRepository; import org.assertj.core.util.Lists; import org.bson.BsonArray; import org.bson.BsonDocument; @@ -245,6 +247,10 @@ public abstract class AbstractPersistenceServerServiceTest { protected MongoTestConfig mongoTestConfig; @MockBean protected ReportTemplatePlaceholderClient reportTemplatePlaceholderClient; + @Autowired + protected CommentRepository commentRepository; + @Autowired + protected ResizeRedactionRepository resizeRedactionRepository; private static String[] getAllRoles() { @@ -474,6 +480,8 @@ public abstract class AbstractPersistenceServerServiceTest { TenantContext.setTenantId(tenant.getTenantId()); + commentRepository.deleteAll(); + resizeRedactionRepository.deleteAll(); falsePositiveEntryRepository.deleteAll(); falseRecommendationEntryRepository.deleteAll(); entryRepository.deleteAll();