From 220d5fa07518257608ac8fb866ff03ae0b330810 Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Tue, 16 Nov 2021 17:35:01 +0100 Subject: [PATCH 1/2] RED-2775 Bugfix, uploading deleted file will ignore excluded pages --- .../FileStatusPersistenceService.java | 10 +- .../service/FileTesterAndProvider.java | 1 - .../v1/server/integration/tests/FileTest.java | 116 +++++++++++++++--- 3 files changed, 105 insertions(+), 22 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 9ebc3d7cc..5e4281b14 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -186,12 +186,6 @@ public class FileStatusPersistenceService { throw new NotFoundException("Unknown file=" + fileId); }); -// if (isFileDeleted(fileId)) { -// return; -// } -// fileRepository.setExcludedPages(fileId, -// OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), -// OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), excludedPages); } @@ -289,6 +283,10 @@ public class FileStatusPersistenceService { @Transactional public void overwriteFile(String fileId, String uploader, String filename) { + fileRepository.findById(fileId).ifPresent((file) -> { + file.setExcludedPages(new HashSet<>()); + }); + int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.UNASSIGNED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index b7a656743..48cb02df9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -45,7 +45,6 @@ public class FileTesterAndProvider { return testAndProvideFile(dossier, "test.pdf"); } - @SneakyThrows public FileModel testAndProvideFile(Dossier dossier, String fileName) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 6bfa2b6da..dbaff073b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -1,5 +1,12 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.util.Collections; + +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; import com.google.common.collect.Sets; import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; @@ -10,17 +17,17 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + import feign.FeignException; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import java.nio.charset.StandardCharsets; - -import static org.assertj.core.api.Assertions.assertThat; public class FileTest extends AbstractPersistenceServerServiceTest { @@ -47,6 +54,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { @Test public void testEmptyFile() { + var dossier = dossierTesterAndProvider.provideTestDossier(); BinaryFileRequest upload = new BinaryFileRequest("".getBytes(StandardCharsets.UTF_8), "emptyFile.pdf", dossier.getId(), "1"); @@ -57,8 +65,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest { } } + @Test public void testFileSoftDeleteReupload() { + var filename = "test.pdf"; var dossier = dossierTesterAndProvider.provideTestDossier(); @@ -82,8 +92,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getFilename()).isEqualTo(filename); } + @Test public void testFile() { + var dossier = dossierTesterAndProvider.provideTestDossier(); var file = fileTesterAndProvider.testAndProvideFile(dossier); @@ -176,9 +188,9 @@ public class FileTest extends AbstractPersistenceServerServiceTest { activeFiles = fileClient.getDossierStatus(dossier.getId()); assertThat(activeFiles.size()).isEqualTo(0); - } + @Test public void testFileWithRedactions() { @@ -199,17 +211,45 @@ public class FileTest extends AbstractPersistenceServerServiceTest { JSONPrimitive uploadResult = uploadClient.upload(upload); var secondFile = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue()); - - var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder().addToDictionary(true) - .addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId).user("user").reason("1").value("test").legalBasis("1").build()); + var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder() + .addToDictionary(true) + .addToDossierDictionary(false) + .comment("comment") + .status(AnnotationStatus.REQUESTED) + .typeId(typeId) + .user("user") + .reason("1") + .value("test") + .legalBasis("1") + .build()); var removeRedaction = manualRedactionClient.addRemoveRedaction(dossierId, fileId, RemoveRedactionRequest.builder() - .annotationId("removeRedactionAnnotation").comment("comment").status(AnnotationStatus.REQUESTED).user("test").removeFromDictionary(false).build()); + .annotationId("removeRedactionAnnotation") + .comment("comment") + .status(AnnotationStatus.REQUESTED) + .user("test") + .removeFromDictionary(false) + .build()); var forceRedaction = manualRedactionClient.addForceRedaction(dossierId, fileId, ForceRedactionRequest.builder() - .annotationId("forceRedactionAnnotation").comment("comment").status(AnnotationStatus.REQUESTED).user("test").legalBasis("1").build()); + .annotationId("forceRedactionAnnotation") + .comment("comment") + .status(AnnotationStatus.REQUESTED) + .user("test") + .legalBasis("1") + .build()); var legalBasisChange = manualRedactionClient.addLegalBasisChange(dossierId, fileId, LegalBasisChangeRequest.builder() - .annotationId("legalBasisChangeAnnotation").comment("comment").status(AnnotationStatus.REQUESTED).user("test").legalBasis("1").build()); + .annotationId("legalBasisChangeAnnotation") + .comment("comment") + .status(AnnotationStatus.REQUESTED) + .user("test") + .legalBasis("1") + .build()); var imageRecategorization = manualRedactionClient.addImageRecategorization(dossierId, fileId, ImageRecategorizationRequest.builder() - .annotationId("imageRecategorizationAnnotation").comment("comment").status(AnnotationStatus.REQUESTED).user("test").typeId("new-type:id").build()); + .annotationId("imageRecategorizationAnnotation") + .comment("comment") + .status(AnnotationStatus.REQUESTED) + .user("test") + .typeId("new-type:id") + .build()); var loadedFile = fileClient.getFileStatus(dossierId, fileId); @@ -274,4 +314,50 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.isExcluded()).isFalse(); assertThat(manualRedactionClient.getAddRedaction(fileId, addRedaction.getAnnotationId()).getFileId()).isEqualTo(loadedFile.getId()); } + + + @Test + public void testFileDeleteReupload() { + + // Upload file + var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(); + + var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate); + String dossierId = dossier.getId(); + + String fileName = "may_the_force_with_you.pdf"; + var file = fileTesterAndProvider.testAndProvideFile(dossier, fileName); + String fileId = file.getId(); + + var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual"); + String typeId = type.getId(); + + assertThat(fileClient.getAllStatuses().size()).isEqualTo(1); + + fileClient.excludePages(dossierId, fileId, Collections.singleton(1)); + assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages().size()).isEqualTo(1); + assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages()).contains(1); + + // Delete file + uploadClient.deleteFile(dossierId, fileId); + var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossierId); + assertThat(softDeletedFiles.size()).isEqualTo(1); + var activeFiles = fileClient.getDossierStatus(dossierId); + assertThat(activeFiles.size()).isEqualTo(0); + + uploadClient.hardDeleteFiles(dossierId, Sets.newHashSet(fileId)); + softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossierId); + assertThat(softDeletedFiles.size()).isEqualTo(0); + + activeFiles = fileClient.getDossierStatus(dossierId); + assertThat(activeFiles.size()).isEqualTo(0); + + // Reupload file + file = fileTesterAndProvider.testAndProvideFile(dossier, fileName); + + assertThat(file.getId()).isEqualTo(fileId); + assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages()).isNullOrEmpty(); + + } + } From 9ca9bf80b382e09f853d7bbd8bd7a7942c918a4a Mon Sep 17 00:00:00 2001 From: Philipp Schramm Date: Wed, 17 Nov 2021 10:05:25 +0100 Subject: [PATCH 2/2] RED-2775 Removed unused and not working code --- .../service/persistence/repository/FileRepository.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 96b354f28..2d2a68941 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -79,12 +79,6 @@ public interface FileRepository extends JpaRepository { void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastManualRedaction, boolean hasSuggestions); - @Modifying - @Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastManualRedaction = :lastManualRedaction, " + - "f.excludedPages = :excludedPages where f.id = :fileId") - void setExcludedPages(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastManualRedaction, - Set excludedPages); - @Modifying @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id = :fileId")