Pull request #412: RED-3900: INC12775037 As a user I want to keep manual redactions if I re-upload a file

Merge in RED/persistence-service from RED-3900-ps5 to master

* commit '389bd6a35fb006bc93085cf5a5fa6c32b4cbf79e':
  RED-3900: INC12775037 As a user I want to keep manual redactions if I re-upload a file

persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java

@@ -29,7 +29,7 @@ public interface UploadResource {
 
     @PostMapping(value = UPLOAD_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces =
             MediaType.APPLICATION_JSON_VALUE)
-    JSONPrimitive<String> upload(@RequestBody AddFileRequest request);
+    JSONPrimitive<String> upload(@RequestBody AddFileRequest request, @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions);
 
 
     @DeleteMapping(value = DELETE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -300,14 +300,22 @@ public class FileStatusPersistenceService {
 
 
     @Transactional
-    public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights) {
+    public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) {
 
-        fileRepository.findById(fileId).ifPresent((file) -> {
+        int countUpdate = 0;
-            file.setExcludedPages(new HashSet<>());
+
-        });
+        if(keepManualRedactions) {
+            countUpdate = fileRepository.overwriteFileAndKeepManualRedactions(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, OffsetDateTime.now()
+                    .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
+        } else {
+
+            fileRepository.findById(fileId).ifPresent((file) -> {
+                file.setExcludedPages(new HashSet<>());
+            });
+
+            countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
+        }
 
-        int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now()
-                .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
         if (countUpdate == 0) {
             throw new NotFoundException("Unknown file=" + fileId);
         }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -133,6 +133,17 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded,
                       OffsetDateTime lastUpdated, boolean hasHighlights);
 
+    @Modifying(clearAutomatically = true)
+    @Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, "
+            + "f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, "
+            + "f.lastOCRTime = null, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, "
+            + "f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null, "
+            + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, "
+            + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, "
+            + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " + "where f.id = :fileId")
+    int overwriteFileAndKeepManualRedactions(String fileId, String filename, String uploader, ProcessingStatus processingStatus, OffsetDateTime lastUploaded,
+                      OffsetDateTime lastUpdated, boolean hasHighlights);
+
     @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where d.dossierTemplateId = :dossierTemplateId" +
             " and ((f.deleted is not null and f.hardDeletedTime is null) or " +
             " (d.softDeletedTime is not null and d.hardDeletedTime is null)) and d.archivedTime is null")

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java

@@ -11,6 +11,7 @@ import com.iqser.red.service.persistence.service.v1.api.resources.UploadResource
 import lombok.RequiredArgsConstructor;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.transaction.Transactional;
@@ -29,9 +30,9 @@ public class UploadController implements UploadResource {
     private final DossierService dossierService;
 
     @Override
-    public JSONPrimitive<String> upload(@RequestBody AddFileRequest request) {
+    public JSONPrimitive<String> upload(@RequestBody AddFileRequest request, @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions) {
 
-        return fileService.upload(request);
+        return fileService.upload(request, keepManualRedactions);
     }
 
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java

@@ -49,7 +49,7 @@ public class FileService {
     private final IndexingService indexingService;
 
 
-    public JSONPrimitive<String> upload(AddFileRequest request) {
+    public JSONPrimitive<String> upload(AddFileRequest request, boolean keepManualRedactions) {
 
         dossierPersistenceService.getAndValidateDossier(request.getDossierId());
 
@@ -79,7 +79,7 @@ public class FileService {
         if (existingStatus != null) {
             // the file is already uploaded, just reanalyse it.
 
-            fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights());
+            fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights(), keepManualRedactions);
         } else {
             // the file is new, should create a new status for it.
             log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -9,7 +9,6 @@ import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
 import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
 import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
 import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
-import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
@@ -436,17 +435,29 @@ public class FileStatusService {
     }
 
 
-    public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
+    public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights, boolean keepManualRedactions) {
+
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.SECTION_GRID);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.IMAGE_INFO);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.TEXT);
+        fileManagementStorageService.deleteObject(dossierId, fileId, FileType.NER_ENTITIES);
+
+        if(keepManualRedactions) {
+            fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, true);
+        } else {
+            fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights, false);
+            deleteManualRedactions(dossierId, fileId);
+        }
 
-        fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights);
-        wipeFileData(dossierId, fileId);
         viewedPagesPersistenceService.deleteForFile(fileId);
+
         setStatusFullReprocess(dossierId, fileId, false, true);
     }
 
 
     @Transactional
-    public void wipeFileData(String dossierId, String fileId) {
+    public void deleteManualRedactions(String dossierId, String fileId) {
 
         OffsetDateTime now = OffsetDateTime.now();
         // remove everything related to redaction

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java

@@ -71,7 +71,7 @@ public class FileTesterAndProvider {
 
         AddFileRequest upload = new AddFileRequest(fileName, fileId, dossier.getId(), "1");
         fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, "test".getBytes(StandardCharsets.UTF_8));
-        JSONPrimitive<String> uploadResult = uploadClient.upload(upload);
+        JSONPrimitive<String> uploadResult = uploadClient.upload(upload, false);
 
         return uploadResult.getValue();
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java

@@ -7,18 +7,28 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
 import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
+import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
 import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.SectionGrid;
+
 import org.junit.Test;
+import org.mockito.Mockito;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.time.OffsetDateTime;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -51,6 +61,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
     @Autowired
     private ViewedPagesClient viewedPagesClient;
 
+    @Autowired
+    private RedactionLogClient redactionLogClient;
+
+
     @Test
     public void testFileSoftDeleteReupload() {
 
@@ -77,6 +91,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getFilename()).isEqualTo(filename);
     }
 
+
     @Test
     public void testFileOverwrite() {
 
@@ -100,7 +115,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(viewedPages.size()).isEqualTo(1);
 
         AddFileRequest upload = new AddFileRequest(filename, file.getId(), dossier.getId(), "1");
-        JSONPrimitive<String> uploadResult = uploadClient.upload(upload);
+        JSONPrimitive<String> uploadResult = uploadClient.upload(upload, false);
 
         loadedFile = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue());
         assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isFalse();
@@ -109,8 +124,52 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(viewedPages.size()).isEqualTo(0);
     }
 
+
+    @Test
+    public void testFileOverwriteAndKeepManualRedactions() {
+
+        var filename = "test.pdf";
+        var dossier = dossierTesterAndProvider.provideTestDossier();
+
+        var file = fileTesterAndProvider.testAndProvideFile(dossier, filename);
+
+        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), "1");
+        fileClient.excludePages(dossier.getId(), file.getId(), Set.of(1));
+        var redactionLog = redactionLogClient.getRedactionLog(dossier.getId(), file.getId(), null, false, true);
+
+        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
+        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
+        assertThat(loadedFile.getFilename()).isEqualTo(filename);
+
+        fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), "1");
+        fileClient.toggleAutomaticAnalysis(dossier.getId(), file.getId(), true);
+        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
+        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();
+
+        viewedPagesClient.addPage(file.getId(), "1", 1);
+        var viewedPages = viewedPagesClient.getViewedPages(file.getId(), "1");
+
+        assertThat(viewedPages.size()).isEqualTo(1);
+
+        AddFileRequest upload = new AddFileRequest(filename, file.getId(), dossier.getId(), "1");
+        JSONPrimitive<String> uploadResult = uploadClient.upload(upload, true);
+
+        loadedFile = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue());
+        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();
+
+        viewedPages = viewedPagesClient.getViewedPages(file.getId(), "1");
+        assertThat(viewedPages.size()).isEqualTo(0);
+
+        assertThat(loadedFile.getAssignee().equals("1"));
+        assertThat(loadedFile.getWorkflowStatus().equals(file.getWorkflowStatus()));
+        assertThat(loadedFile.getExcludedPages().equals(file.getExcludedPages()));
+
+    }
+
+
     @Test
     public void testFile() {
+
         var start = OffsetDateTime.now();
 
         var dossier = dossierTesterAndProvider.provideTestDossier();
@@ -131,7 +190,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getFileManipulationDate()).isNotNull();
 
-
         fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), "1");
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getAssignee()).isEqualTo("1");
@@ -139,17 +197,14 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getLastReviewer()).isNull();
         assertThat(loadedFile.getLastApprover()).isNull();
 
-
         fileClient.excludePages(dossier.getId(), file.getId(), Sets.newHashSet(1));
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getExcludedPages()).containsExactlyInAnyOrder(1);
 
-
         fileClient.includePages(dossier.getId(), file.getId(), Sets.newHashSet(1));
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getExcludedPages()).isEmpty();
 
-
         fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), "1");
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getAssignee()).isEqualTo("1");
@@ -157,7 +212,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getLastReviewer()).isNull();
         assertThat(loadedFile.getLastApprover()).isNull();
 
-
         fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
@@ -180,7 +234,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getLastApprover()).isEqualTo("1");
         assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();
 
-
         fileClient.setStatusUnderReview(dossier.getId(), file.getId(), "2");
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
@@ -189,7 +242,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getLastApprover()).isEqualTo("1");
         assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();
 
-
         fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), "2");
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL);
@@ -197,7 +249,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getLastReviewer()).isEqualTo("2");
         assertThat(loadedFile.getLastApprover()).isEqualTo("1");
 
-
         fileClient.setStatusApproved(dossier.getId(), file.getId(), "2");
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED);
@@ -205,15 +256,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.getLastReviewer()).isEqualTo("2");
         assertThat(loadedFile.getLastApprover()).isEqualTo("2");
 
-
         fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null);
 
-
         fileClient.toggleExclusion(dossier.getId(), file.getId(), true);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.isExcluded()).isTrue();
 
-
         fileClient.toggleExclusion(dossier.getId(), file.getId(), false);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.isExcluded()).isFalse();
@@ -225,14 +273,12 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         var activeFiles = fileClient.getDossierStatus(dossier.getId());
         assertThat(activeFiles.size()).isEqualTo(0);
 
-
         uploadClient.undeleteFiles(dossier.getId(), Sets.newHashSet(file.getId()));
         softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
         assertThat(softDeletedFiles.size()).isEqualTo(0);
         activeFiles = fileClient.getDossierStatus(dossier.getId());
         assertThat(activeFiles.size()).isEqualTo(1);
 
-
         uploadClient.hardDeleteFiles(dossier.getId(), Sets.newHashSet(file.getId()));
         softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
         assertThat(softDeletedFiles.size()).isEqualTo(0);
@@ -307,7 +353,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(manualRedactionClient.getLegalBasisChange(fileId, legalBasisChange.getAnnotationId()).getFileId()).isEqualTo(loadedFile.getId());
         assertThat(manualRedactionClient.getImageRecategorization(fileId, imageRecategorization.getAnnotationId()).getFileId()).isEqualTo(loadedFile.getId());
 
-
         uploadClient.deleteFile(dossier.getId(), file.getId());
         var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
         assertThat(softDeletedFiles.size()).isEqualTo(1);
@@ -329,8 +374,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
 
     }
 
+
     @Test
     public void testToggleEnableRedactionTwice() {
+
         var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
 
         var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
@@ -344,9 +391,17 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
 
         assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
 
-        var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder().addToDictionary(true)
+        var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, Collections.singletonList(AddRedactionRequest.builder()
-                .addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId)
+                .addToDictionary(true)
-                .user("user").reason("1").value("test").legalBasis("1").build())).iterator().next();
+                .addToDossierDictionary(false)
+                .comment("comment")
+                .status(AnnotationStatus.REQUESTED)
+                .typeId(typeId)
+                .user("user")
+                .reason("1")
+                .value("test")
+                .legalBasis("1")
+                .build())).iterator().next();
 
         var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
 
@@ -357,7 +412,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
         assertThat(loadedFile.isExcluded()).isTrue();
         assertThat(manualRedactionClient.getAddRedaction(fileId, addRedaction.getAnnotationId()).getFileId()).isEqualTo(loadedFile.getId());
 
-
         fileClient.toggleExclusion(dossier.getId(), file.getId(), false);
         loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
         assertThat(loadedFile.isExcluded()).isFalse();