package com.iqser.red.service.peristence.v1.server.integration.tests;

import static com.iqser.red.service.persistence.service.v1.api.external.resource.FileAttributesResource.UTF_ENCODING;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mockito;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.mock.web.MockMultipartFile;

import com.github.dockerjava.zerodep.shaded.org.apache.commons.codec.binary.Base64;
import com.google.common.collect.Sets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileAttributeConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileManagementClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ManualRedactionClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ReanalysisClient;
import com.iqser.red.service.peristence.v1.server.integration.client.UploadClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ViewedPagesClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileDeletionService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DeletedFilesCleanupJob;
import com.iqser.red.service.persistence.management.v1.processor.service.job.DocumentDataFilesConversionJob;
import com.iqser.red.service.persistence.management.v1.processor.service.job.SoftDeletedFilesCleanupJob;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.utils.StorageIdUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributes;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageExclusionRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.ViewedPagesRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationComments;
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AddNotificationRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddCommentRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.AddRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.ForceRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.LegalBasisChangeRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RecategorizationRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.manual.RemoveRedactionRequestModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.notification.NotificationType;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.model.TenantResponse;

import feign.FeignException;
import lombok.SneakyThrows;

public class FileTest extends AbstractPersistenceServerServiceTest {

    @Autowired
    private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;

    @Autowired
    private FileTesterAndProvider fileTesterAndProvider;

    @Autowired
    private DossierTesterAndProvider dossierTesterAndProvider;

    @Autowired
    private TypeProvider typeProvider;

    @Autowired
    private FileClient fileClient;

    @Autowired
    private UploadClient uploadClient;

    @Autowired
    private ManualRedactionClient manualRedactionClient;

    @Autowired
    private DossierClient dossierClient;

    @Autowired
    private ViewedPagesClient viewedPagesClient;

    @Autowired
    private FileAttributeClient fileAttributeClient;

    @Autowired
    private FileAttributeConfigClient fileAttributeConfigClient;

    @Autowired
    private DossierService dossierService;

    @Autowired
    private FileDeletionService fileDeletionService;

    @Autowired
    private FileManagementClient fileManagementClient;

    @Autowired
    private ReanalysisClient reanalysisClient;

    @Autowired
    private UserProvider userProvider;

    @Autowired
    private FileManagementStorageService fileManagementStorageService;

    @Autowired
    private FileStatusService fileStatusService;

    @Autowired
    private DeletedFilesCleanupJob deletedFilesCleanupJob;

    @Autowired
    private SoftDeletedFilesCleanupJob softDeletedFilesCleanupJob;

    @Autowired
    private DocumentDataFilesConversionJob documentDataFilesConversionJob;
    @MockBean
    private NotificationPersistenceService notificationPersistenceService;
    @Captor
    private ArgumentCaptor<AddNotificationRequest> captor;


    @Test
    public void testFileSoftDeleteReupload() {

        var filename = "test";
        var dossier = dossierTesterAndProvider.provideTestDossier();

        var file = fileTesterAndProvider.testAndProvideFile(dossier, filename);

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());

        fileManagementClient.deleteFile(dossier.getId(), file.getId());

        var nrOfFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId()).size();
        assertThat(nrOfFiles).isEqualTo(1);
        loadedFile = fileClient.getSoftDeletedDossierStatus(dossier.getId())
                .get(0);

        var fileUploaded = fileTesterAndProvider.testAndProvideFile(dossier);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());

        assertThat(loadedFile.getFilename()).startsWith(filename);
    }


    @Test
    public void testFileOverwrite() {

        var filename = "test";
        var dossier = dossierTesterAndProvider.provideTestDossier();

        var file = fileTesterAndProvider.testAndProvideFile(dossier, filename);

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());

        var userId = userProvider.getUserId();
        fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), userId);
        reanalysisClient.toggleAutomaticAnalysis(dossier.getId(), file.getId(), true);

        List<FileAttributeConfig> configs = new ArrayList<>();
        configs.add(FileAttributeConfig.builder().csvColumnHeader("Name").primaryAttribute(true).label("Name").build());
        configs.add(FileAttributeConfig.builder().csvColumnHeader("Attribute A").primaryAttribute(true).label("Attribute A").build());
        var loadedConfig = fileAttributeConfigClient.setFileAttributesConfig(dossier.getDossierTemplateId(), new FileAttributesConfig(configs));
        fileAttributeClient.setFileAttributes(dossier.getId(),
                                              file.getId(),
                                              new FileAttributes(Map.of(loadedConfig.getFileAttributeConfigs()
                                                                                .get(0).getId(), "123")));

        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();
        assertThat(loadedFile.getFileAttributes().getAttributeIdToValue()).size().isEqualTo(1);
        assertThat(loadedFile.getFileAttributes().getAttributeIdToValue()).isEqualTo(Map.of(loadedConfig.getFileAttributeConfigs()
                                                                                                    .get(0).getId(), "123"));

        viewedPagesClient.addPage(file.getDossierId(), file.getFileId(), new ViewedPagesRequest(1));
        var viewedPages = viewedPagesClient.getViewedPages(file.getDossierId(), file.getFileId());

        assertThat(viewedPages.getPages().size()).isEqualTo(1);

        var fileUpload = new MockMultipartFile("test.pdf", "test.pdf", "application/pdf", "content".getBytes());
        var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false, false);

        loadedFile = fileClient.getFileStatus(dossier.getId(),
                                              uploadResult.getFileIds()
                                                      .iterator().next());
        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isFalse();
        assertThat(loadedFile.getFileAttributes().getAttributeIdToValue()).size().isEqualTo(0);

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), userId);

        viewedPages = viewedPagesClient.getViewedPages(file.getDossierId(), file.getFileId());
        assertThat(viewedPages.getPages().size()).isEqualTo(0);
    }


    @Test
    public void testFileOverwriteAndKeepManualRedactions() {

        var filename = "test";
        var dossier = dossierTesterAndProvider.provideTestDossier();
        var userId = userProvider.getUserId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier, filename);

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), userId);
        reanalysisClient.excludePages(dossier.getId(), file.getId(), new PageExclusionRequest(List.of(new PageRange(1, 1))));

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());

        fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), userId);
        reanalysisClient.toggleAutomaticAnalysis(dossier.getId(), file.getId(), true);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();

        viewedPagesClient.addPage(file.getDossierId(), file.getFileId(), new ViewedPagesRequest(1));
        var viewedPages = viewedPagesClient.getViewedPages(file.getDossierId(), file.getFileId());

        assertThat(viewedPages.getPages()).hasSize(1);

        var fileUpload = new MockMultipartFile(file.getFilename(), file.getFilename(), "application/pdf", "content".getBytes());
        var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false, false);

        loadedFile = fileClient.getFileStatus(dossier.getId(),
                                              uploadResult.getFileIds()
                                                      .iterator().next());
        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isFalse();

    }


    @Test
    public void testFile() {

        var start = OffsetDateTime.now();

        var dossier = dossierTesterAndProvider.provideTestDossier();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        var userId = userProvider.getUserId();
        var altUserId = userProvider.getAltUserId();

        var changes = dossierClient.changesSince(JSONPrimitive.of(start));
        assertThat(changes).isNotEmpty();
        assertThat(changes.iterator().next().isFileChanges()).isTrue();
        assertThat(changes.iterator().next().isDossierChanges()).isTrue();

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getFileManipulationDate()).isNotNull();

        fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), userId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW);
        assertThat(loadedFile.getLastReviewer()).isNull();
        assertThat(loadedFile.getLastApprover()).isNull();

        reanalysisClient.excludePages(dossier.getId(), file.getId(), new PageExclusionRequest(List.of(new PageRange(1, 1))));
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getExcludedPages()).containsExactlyInAnyOrder(1);

        reanalysisClient.includePages(dossier.getId(), file.getId(), new PageExclusionRequest(List.of(new PageRange(1, 1))));
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getExcludedPages()).isEmpty();

        fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), userId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW);
        assertThat(loadedFile.getLastReviewer()).isNull();
        assertThat(loadedFile.getLastApprover()).isNull();

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isNull();

        fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), userId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL);
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isEqualTo(userId);

        fileTesterAndProvider.markFileAsProcessed(dossier.getId(), file.getFileId());
        fileClient.setStatusApproved(dossier.getId(), file.getId(), true);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED);
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isEqualTo(userId);
        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), altUserId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
        assertThat(loadedFile.getAssignee()).isEqualTo(altUserId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isEqualTo(userId);
        assertThat(loadedFile.isExcludedFromAutomaticAnalysis()).isTrue();

        fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), altUserId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL);
        assertThat(loadedFile.getAssignee()).isEqualTo(altUserId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isEqualTo(altUserId);

        fileClient.setStatusApproved(dossier.getId(), file.getId(), true);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED);
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isEqualTo(altUserId);

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null);

        reanalysisClient.toggleExclusion(dossier.getId(), file.getId(), true);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.isExcluded()).isTrue();

        reanalysisClient.toggleExclusion(dossier.getId(), file.getId(), false);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.isExcluded()).isFalse();

        fileManagementClient.deleteFile(dossier.getId(), file.getId());

        var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
        assertThat(softDeletedFiles.size()).isEqualTo(1);
        var activeFiles = fileClient.getDossierStatus(dossier.getId());
        assertThat(activeFiles.size()).isEqualTo(0);

        fileManagementClient.restoreFiles(dossier.getId(), Sets.newHashSet(file.getId()));
        softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
        assertThat(softDeletedFiles.size()).isEqualTo(0);
        activeFiles = fileClient.getDossierStatus(dossier.getId());
        assertThat(activeFiles.size()).isEqualTo(1);

        fileManagementClient.hardDeleteFiles(dossier.getId(), List.of(file.getId()));
        softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
        assertThat(softDeletedFiles.size()).isEqualTo(0);

        activeFiles = fileClient.getDossierStatus(dossier.getId());
        assertThat(activeFiles.size()).isEqualTo(0);

    }


    @Test
    public void testFileWithRedactions() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2");
        typeProvider.testAndProvideType(dossierTemplate, null, "new-type", false, 2000);
        var annotationId = "imagine_this_makes_sense";
        var commentId = "commentId";
        EntityLog entityLog = new EntityLog(1,
                                            1,
                                            List.of(EntityLogEntry.builder()
                                                            .id(annotationId)
                                                            .type(type.getType())
                                                            .value("value entry")
                                                            .state(EntryState.APPLIED)
                                                            .entryType(EntryType.ENTITY)
                                                            .build(),
                                                    EntityLogEntry.builder()
                                                            .id("forceRedactionAnnotation")
                                                            .type(type.getType())
                                                            .value("value entry 2")
                                                            .state(EntryState.APPLIED)
                                                            .entryType(EntryType.ENTITY)
                                                            .build(),
                                                    EntityLogEntry.builder()
                                                            .id("legalBasisChangeAnnotation")
                                                            .type(type.getType())
                                                            .value("value entry 3")
                                                            .state(EntryState.APPLIED)
                                                            .entryType(EntryType.ENTITY)
                                                            .build(),
                                                    EntityLogEntry.builder()
                                                            .id(commentId)
                                                            .type(type.getType())
                                                            .value("value entry 4")
                                                            .dictionaryEntry(true)
                                                            .state(EntryState.APPLIED)
                                                            .entryType(EntryType.ENTITY)
                                                            .build()),
                                            null,
                                            0,
                                            0,
                                            0,
                                            0);

        fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
        when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);

        var addRedactionRequest = AddRedactionRequestModel.builder()
                .addToDictionary(true)
                .type(type.getType())
                .addToAllDossiers(true)
                .reason("1")
                .value("test")
                .legalBasis("1")
                .dictionaryEntryType(DictionaryEntryType.ENTRY)
                .build();
        manualRedactionClient.addRedactionBulk(dossierId, fileId, Set.of(addRedactionRequest));

        manualRedactionClient.removeRedactionBulk(dossierId,
                                                  fileId,
                                                  Set.of(RemoveRedactionRequestModel.builder().annotationId(annotationId).comment("removeComment").removeFromDictionary(false).build()));
        manualRedactionClient.forceRedactionBulk(dossierId,
                                                 fileId,
                                                 Set.of(ForceRedactionRequestModel.builder().annotationId("forceRedactionAnnotation").comment("forceComment").legalBasis("1").build()));
        manualRedactionClient.legalBasisChangeBulk(dossierId,
                                                   fileId,
                                                   Set.of(LegalBasisChangeRequestModel.builder()
                                                                  .annotationId("legalBasisChangeAnnotation")
                                                                  .comment("legalBasisComment")
                                                                  .legalBasis("1")
                                                                  .build()));
        manualRedactionClient.recategorizeBulk(dossierId,
                                               fileId,
                                               Set.of(RecategorizationRequestModel.builder()
                                                              .annotationId(annotationId)
                                                              .comment("recategorizationComment")
                                                              .type("new-type")
                                                              .legalBasis(null)
                                                              .section("section")
                                                              .build()));

        manualRedactionClient.addComment(dossierId, fileId, commentId, AddCommentRequestModel.builder().text("comment added via text fileId").build());

        var loadedFile = fileClient.getFileStatus(dossierId, fileId);

        //check comments are restored
        AnnotationComments annotationComments = manualRedactionClient.getComments(dossierId, fileId, commentId);
        assertThat(annotationComments.getComments()).hasSize(1);
        AnnotationComments annotationCommentsForManualRedactions = manualRedactionClient.getComments(dossierId, fileId, annotationId);
        assertThat(annotationCommentsForManualRedactions.getComments()).hasSize(1);

        fileManagementClient.deleteFile(dossier.getId(), file.getId());
        var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
        assertThat(softDeletedFiles).hasSize(1);
        var activeFiles = fileClient.getDossierStatus(dossier.getId());
        assertThat(activeFiles).isEmpty();

        // check the comment added to an annotation that has no manual redaction
        Exception exception = Assertions.assertThrows(FeignException.NotFound.class, () -> manualRedactionClient.getComments(dossierId, fileId, commentId));

        String expectedMessage = "The requested file has been soft deleted on";
        String actualMessage = exception.getMessage();
        assertThat(actualMessage).contains(expectedMessage);

        fileManagementClient.restoreFiles(dossier.getId(), Sets.newHashSet(file.getId()));
        softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
        assertThat(softDeletedFiles).isEmpty();
        activeFiles = fileClient.getDossierStatus(dossier.getId());
        assertThat(activeFiles).hasSize(1);
        //check comments are restored
        annotationComments = manualRedactionClient.getComments(dossierId, fileId, commentId);
        assertThat(annotationComments.getComments()).hasSize(1);
        annotationCommentsForManualRedactions = manualRedactionClient.getComments(dossierId, fileId, annotationId);
        assertThat(annotationCommentsForManualRedactions.getComments()).hasSize(1);

        fileManagementClient.hardDeleteFiles(dossier.getId(), List.of(file.getId()));
        softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
        assertThat(softDeletedFiles).isEmpty();

        activeFiles = fileClient.getDossierStatus(dossier.getId());
        assertThat(activeFiles).isEmpty();

    }


    @Test
    public void testToggleEnableRedactionTwice() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2");

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);

        var userId = userProvider.getUserId();

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), userId);

        manualRedactionClient.addRedactionBulk(dossierId,
                                               fileId,
                                               Set.of(AddRedactionRequestModel.builder()
                                                              .addToDictionary(true)
                                                              .addToAllDossiers(true)
                                                              .comment(new AddCommentRequestModel("comment"))
                                                              .type(type.getType())
                                                              .reason("1")
                                                              .value("test")
                                                              .legalBasis("1")
                                                              .dictionaryEntryType(DictionaryEntryType.ENTRY)
                                                              .build()));

        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());

        reanalysisClient.toggleExclusion(dossier.getId(), file.getId(), true);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.isExcluded()).isTrue();

        reanalysisClient.toggleExclusion(dossier.getId(), file.getId(), false);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.isExcluded()).isFalse();
    }


    @Test
    public void testFileDeleteReupload() {

        // Upload file
        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        String fileName = "may_the_force_with_you.pdf";
        var file = fileTesterAndProvider.testAndProvideFile(dossier, fileName);
        String fileId = file.getId();

        var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2");

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);

        reanalysisClient.excludePages(dossierId, fileId, new PageExclusionRequest(List.of(new PageRange(1, 1))));
        assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages().size()).isEqualTo(1);
        assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages()).contains(1);

        // Delete file
        fileManagementClient.deleteFile(dossierId, fileId);
        var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossierId);
        assertThat(softDeletedFiles.size()).isEqualTo(1);
        var activeFiles = fileClient.getDossierStatus(dossierId);
        assertThat(activeFiles.size()).isEqualTo(0);

        fileManagementClient.hardDeleteFiles(dossierId, List.of(fileId));
        softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossierId);
        assertThat(softDeletedFiles.size()).isEqualTo(0);

        activeFiles = fileClient.getDossierStatus(dossierId);
        assertThat(activeFiles.size()).isEqualTo(0);

        // Reupload file
        file = fileTesterAndProvider.testAndProvideFile(dossier, fileName);

        assertThat(file.getId()).isEqualTo(fileId);
        assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages()).isNullOrEmpty();

    }


    @Test
    public void testFile4424() {

        var start = OffsetDateTime.now();
        var dossier = dossierTesterAndProvider.provideTestDossier();
        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        var userId = userProvider.getUserId();
        var altUserId = userProvider.getAltUserId();

        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getAssignee()).isNull();
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW);
        assertThat(loadedFile.getLastReviewer()).isNull();
        assertThat(loadedFile.getLastApprover()).isNull();

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), userId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getLastReviewer()).isNull();
        assertThat(loadedFile.getLastApprover()).isNull();

        fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), altUserId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
        assertThat(loadedFile.getAssignee()).isEqualTo(altUserId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isNull();

        fileClient.setStatusUnderApproval(dossier.getId(), file.getId(), altUserId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL);
        assertThat(loadedFile.getAssignee()).isEqualTo(altUserId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isEqualTo(altUserId);

        fileClient.setCurrentFileAssignee(dossier.getId(), file.getId(), userId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_APPROVAL);
        assertThat(loadedFile.getAssignee()).isEqualTo(userId);
        assertThat(loadedFile.getLastReviewer()).isEqualTo(userId);
        assertThat(loadedFile.getLastApprover()).isEqualTo(altUserId);

    }


    @Test
    public void testFile6034SetUnderReview() {

        var start = OffsetDateTime.now();
        var dossier = dossierTesterAndProvider.provideTestDossier();
        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        var userId = userProvider.getUserId();
        var altUserId = userProvider.getAltUserId();
        var user2 = userProvider.getMemberUserId();

        assertThat(dossier.getMemberIds().contains(user2)).isFalse();

        var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getAssignee()).isNull();
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW);
        assertThat(loadedFile.getLastReviewer()).isNull();
        assertThat(loadedFile.getLastApprover()).isNull();

        Exception exception = assertThrows(FeignException.BadRequest.class, () -> fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null));

        String expectedMessage = "File is already unassigned!";
        String actualMessage = exception.getMessage();

        assertThat(actualMessage).contains(expectedMessage);

        exception = assertThrows(FeignException.BadRequest.class, () -> fileClient.setStatusUnderReview(dossier.getId(), file.getId(), user2));

        expectedMessage = "User must be dossier member";
        actualMessage = exception.getMessage();

        assertThat(actualMessage).contains(expectedMessage);

        fileClient.setStatusUnderReview(dossier.getId(), file.getId(), altUserId);
        loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertThat(loadedFile.getWorkflowStatus()).isEqualTo(WorkflowStatus.UNDER_REVIEW);
        assertThat(loadedFile.getAssignee()).isEqualTo(altUserId);
        assertThat(loadedFile.getLastReviewer()).isNull();
        assertThat(loadedFile.getLastApprover()).isNull();
    }


    @Test
    @SneakyThrows
    public void testUploadFileIsInOcrProcessingWithOcrByDefaultFlagTrue() {

        DossierTemplateModel dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate(true);
        Dossier dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "1");
        String fileName = "test";
        HashFunction hashFunction = Hashing.murmur3_128();
        String fileId = hashFunction.hashBytes(((fileName + ".pdf") + dossier.getId()).getBytes(StandardCharsets.UTF_8)).toString();

        fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.ORIGIN, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
        MockMultipartFile mff = new MockMultipartFile(fileName, fileName + ".pdf", "application/pdf", "lorem ipsum".getBytes());
        FileUploadResult uploadResult = uploadClient.upload(mff, dossier.getId(), false, false);

        fileId = uploadResult.getFileIds()
                .iterator().next();
        FileStatus file = fileClient.getFileStatus(dossier.getId(), fileId);

        assertThat(file.getProcessingStatus()).isEqualTo(ProcessingStatus.OCR_PROCESSING_QUEUED);
    }


    @Test
    public void testGetFileNameQuery() {

        var filename = "test";
        var dossier = dossierTesterAndProvider.provideTestDossier();

        var file = fileTesterAndProvider.testAndProvideFile(dossier, filename);

        assertThat(fileStatusService.getFileName(file.getId())).isEqualTo(filename + ".pdf");
    }


    @Test
    public void testUploadWronglyFormattedCsv() {

        FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService = Mockito.mock(FileAttributeConfigPersistenceService.class);
        String fileName = "file";
        var dossier = dossierTesterAndProvider.provideTestDossier();
        String malformedCsvContent = "Path,\"Document Title\",\"Major Version Number\",\"Minor Version Number\",\"Vertebrate Study\"\n\"402Study.pdf\",\"My Title\",\"4.636.0,\"4.363.0\",\"4.363.0\"";
        var fileId = Base64.encodeBase64String((dossier.getId() + fileName).getBytes(StandardCharsets.UTF_8));
        var malformedCsvFile = new MockMultipartFile(fileName + ".csv", fileName + ".csv", "text/csv", malformedCsvContent.getBytes());

        fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, new ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
        when(fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(anyString())).thenReturn(FileAttributesGeneralConfigurationEntity.builder()
                                                                                                                          .delimiter(",")
                                                                                                                          .encoding(UTF_ENCODING)
                                                                                                                          .build());
        when(fileAttributeConfigPersistenceService.getFileAttributes(anyString())).thenReturn(Collections.emptyList());
        assertThrows(FeignException.class, () -> uploadClient.upload(malformedCsvFile, dossier.getId(), false, false));
    }


    @Test
    void testFileWithCommentsAtReupload() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2"); // manual is a system managed type automatcally created
        typeProvider.testAndProvideType(dossierTemplate, null, "new-type", false, 2000);
        var annotationId = "imagine_this_makes_sense";
        var annotation2Id = "imagine_this_makes_sense 2";
        EntityLog entityLog = new EntityLog(1,
                                            1,
                                            List.of(EntityLogEntry.builder()
                                                            .id(annotationId)
                                                            .type(type.getType())
                                                            .value("value entry")
                                                            .state(EntryState.APPLIED)
                                                            .entryType(EntryType.ENTITY)
                                                            .build(),
                                                    EntityLogEntry.builder()
                                                            .id(annotation2Id)
                                                            .type(type.getType())
                                                            .value("value entry 3")
                                                            .state(EntryState.APPLIED)
                                                            .entryType(EntryType.ENTITY)
                                                            .build()),
                                            null,
                                            0,
                                            0,
                                            0,
                                            0);

        fileManagementStorageService.saveEntityLog(dossier.getId(), file.getId(), entityLog);
        when(entityLogService.getEntityLog(any(), any(), anyBoolean())).thenReturn(entityLog);

        assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);

        var recatResponse = manualRedactionClient.recategorizeBulk(dossierId,
                                                                   fileId,
                                                                   Set.of(RecategorizationRequestModel.builder()
                                                                                  .annotationId(annotation2Id)
                                                                                  .comment("comment edit via edit dialog")
                                                                                  .type(type.getType())
                                                                                  .legalBasis("")
                                                                                  .section("section")
                                                                                  .value("value entry 3")
                                                                                  .build()));

        var loadedFile = fileClient.getFileStatus(dossierId, fileId);

        manualRedactionClient.addComment(dossierId, fileId, annotationId, AddCommentRequestModel.builder().text("comment added via text fileId").build());

        //check comments are restored
        AnnotationComments annotationComments = manualRedactionClient.getComments(dossierId, fileId, annotationId);
        assertThat(annotationComments.getComments()).hasSize(1);
        AnnotationComments annotationCommentsForInitialAnnotation = manualRedactionClient.getComments(dossierId, fileId, annotation2Id);
        assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0);
        AnnotationComments annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId,
                                                                                                fileId,
                                                                                                recatResponse.getManualAnnotationResponses()
                                                                                                        .get(0).getAnnotationId());
        assertThat(annotationCommentsForManualRecat.getComments()).hasSize(1);

        //overwrite the file
        var fileUpload = new MockMultipartFile(file.getFilename(), file.getFilename(), "application/pdf", "content".getBytes());
        var uploadResult = uploadClient.upload(fileUpload, dossier.getId(), false, false);

        loadedFile = fileClient.getFileStatus(dossier.getId(),
                                              uploadResult.getFileIds()
                                                      .iterator().next());

        //check comments are soft deleted
        annotationComments = manualRedactionClient.getComments(dossierId, loadedFile.getId(), annotation2Id);
        assertThat(annotationComments.getComments()).hasSize(0);
        annotationCommentsForInitialAnnotation = manualRedactionClient.getComments(dossierId, fileId, annotation2Id);
        assertThat(annotationCommentsForInitialAnnotation.getComments()).hasSize(0);
        annotationCommentsForManualRecat = manualRedactionClient.getComments(dossierId,
                                                                             fileId,
                                                                             recatResponse.getManualAnnotationResponses()
                                                                                     .get(0).getAnnotationId());
        assertThat(annotationCommentsForManualRecat.getComments()).hasSize(0);

    }


    @Test
    @Disabled
    public void testSoftDeletedFilesCleanupJob() {

        OffsetDateTime enoughTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getSoftDeleteCleanupTime() / 24);

        FileModel fileModel = executeSoftDeletedFilesCleanupJobForGivenTime(enoughTimePassed);

        assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getHardDeletedTime());
        assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getSoftDeletedTime());
        assertNotNull(fileModel.getHardDeletedTime());
        assertNotNull(fileModel.getDeleted());

    }


    @Test
    @Disabled
    public void testSoftDeletedFilesCleanupJobUnsuccessful() {

        OffsetDateTime notEnoughTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getSoftDeleteCleanupTime() / 24).plusHours(1);

        FileModel fileModel = executeSoftDeletedFilesCleanupJobForGivenTime(notEnoughTimePassed);

        assertNull(dossierService.getDossierById(fileModel.getDossierId()).getHardDeletedTime());
        assertNotNull(dossierService.getDossierById(fileModel.getDossierId()).getSoftDeletedTime());
        assertNull(fileModel.getHardDeletedTime());
        assertNotNull(fileModel.getDeleted());
    }


    private FileModel executeSoftDeletedFilesCleanupJobForGivenTime(OffsetDateTime offsetDateTime) {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        var otherFile = fileTesterAndProvider.testAndProvideFile(dossier, "otherFile");
        String otherFileId = otherFile.getId();

        dossierService.softDeleteDossier(dossierId, offsetDateTime);
        fileDeletionService.softDeleteFiles(List.of(fileId, otherFileId), offsetDateTime);
        fileDeletionService.hardDeleteFile(dossierId, otherFileId);

        when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));

        assertNull(dossierService.getDossierById(dossierId).getHardDeletedTime());
        assertNotNull(dossierService.getDossierById(dossierId).getSoftDeletedTime());
        assertNull(fileStatusService.getDossierStatus(dossierId)
                           .get(0).getHardDeletedTime());
        assertNotNull(fileStatusService.getDossierStatus(dossierId)
                              .get(0).getDeleted());
        assertNotNull(fileStatusService.getDossierStatus(dossierId)
                              .get(1).getHardDeletedTime());
        assertNotNull(fileStatusService.getDossierStatus(dossierId)
                              .get(1).getDeleted());

        softDeletedFilesCleanupJob.execute(null);

        Optional<FileModel> otherFileModel = fileStatusService.getDossierStatus(dossierId)
                .stream()
                .filter(fm -> fm.getId().equals(otherFileId))
                .findFirst();
        Assertions.assertTrue(otherFileModel.isPresent());

        assertNotNull(otherFileModel.get().getHardDeletedTime());
        assertNotNull(otherFileModel.get().getDeleted());

        Optional<FileModel> fileModel = fileStatusService.getDossierStatus(dossierId)
                .stream()
                .filter(fm -> fm.getId().equals(fileId))
                .findFirst();
        Assertions.assertTrue(fileModel.isPresent());
        return fileModel.get();
    }


    @Test
    public void testHardDeletedFilesCleanupJob() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        OffsetDateTime withinRetryTime = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getHardDeleteCleanupRetryTime() / 24).plusHours(1);

        executeDeletedFilesCleanupJobForGivenTime(withinRetryTime, dossierId, fileId);

        Assertions.assertFalse(fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN));

    }


    @Test
    public void testHardDeletedFilesCleanupJobUnsuccessful() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        OffsetDateTime tooMuchTimePassed = OffsetDateTime.now().minusDays(appConfigClient.getCurrentApplicationConfig().getHardDeleteCleanupRetryTime() / 24).minusHours(1);

        executeDeletedFilesCleanupJobForGivenTime(tooMuchTimePassed, dossierId, fileId);

        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.ORIGIN));

    }


    private void executeDeletedFilesCleanupJobForGivenTime(OffsetDateTime offsetDateTime, String dossierId, String fileId) {

        byte[] bytesToStore = {1, 2, 3, 4};
        fileManagementStorageService.storeObject(dossierId, fileId, FileType.ORIGIN, new ByteArrayInputStream(bytesToStore));

        dossierService.hardDeleteDossier(dossierId, offsetDateTime);
        fileDeletionService.hardDeleteFiles(dossierId, List.of(fileId), offsetDateTime);

        when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));

        assertNotNull(dossierService.getDossierById(dossierId).getHardDeletedTime());
        assertNotNull(fileStatusService.getDossierStatus(dossierId)
                              .get(0).getHardDeletedTime());
        Assertions.assertArrayEquals(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN), bytesToStore);

        deletedFilesCleanupJob.execute(null);
    }


    @Test
    @SneakyThrows
    public void testDocumentDataFilesConversionJobSuccessful() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE.name(), ".json");
        String newDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
        String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES.name(), ".json");
        String newDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
        String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION.name(), ".json");
        String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
        String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT.name(), ".json");
        String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);

        ClassPathResource oldDocumentStructureClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_STRUCTURE.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentStructureStorageId, oldDocumentStructureClassPath.getInputStream());
        ClassPathResource oldDocumentPagesClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_PAGES.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentPagesStorageId, oldDocumentPagesClassPath.getInputStream());
        ClassPathResource oldDocumentPositionDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_POSITION.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId, oldDocumentPositionDataClassPath.getInputStream());
        ClassPathResource oldDocumentTextDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_TEXT.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId, oldDocumentTextDataClassPath.getInputStream());

        when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
        documentDataFilesConversionJob.execute(null);

        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE));
        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_TEXT));
        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_PAGES));
        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_POSITION));

        DocumentStructureProto.DocumentStructure documentStructure = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                    newDocumentStructureStorageId,
                                                                                                    DocumentStructureProto.DocumentStructure.parser());
        assertEquals(documentStructure.getRoot().getPageNumbersList().size(), 9);

        DocumentPageProto.AllDocumentPages allDocumentPages = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                             newDocumentPagesStorageId,
                                                                                             DocumentPageProto.AllDocumentPages.parser());
        assertEquals(allDocumentPages.getDocumentPagesList().size(), 9);

        DocumentTextDataProto.AllDocumentTextData allDocumentTextData = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                       newDocumentTextDataStorageId,
                                                                                                       DocumentTextDataProto.AllDocumentTextData.parser());
        assertEquals(allDocumentTextData.getDocumentTextDataCount(), 114);

        DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                                   newDocumentPositionDataStorageId,
                                                                                                                   DocumentPositionDataProto.AllDocumentPositionData.parser());
        assertEquals(allDocumentPositionData.getDocumentPositionDataList().size(), 114);

    }


    @Test
    @SneakyThrows
    @Disabled
    public void serializeDocumentDataFiles() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();

        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();

        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        String newDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
        String newDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
        String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
        String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);

        ClassPathResource newDocumentStructureClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_STRUCTURE.proto");
        storageService.storeObject(TenantContext.getTenantId(), newDocumentStructureStorageId, newDocumentStructureClassPath.getInputStream());
        ClassPathResource newDocumentPagesClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_PAGES.proto");
        storageService.storeObject(TenantContext.getTenantId(), newDocumentPagesStorageId, newDocumentPagesClassPath.getInputStream());
        ClassPathResource newDocumentPositionDataClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_POSITION.proto");
        storageService.storeObject(TenantContext.getTenantId(), newDocumentPositionDataStorageId, newDocumentPositionDataClassPath.getInputStream());
        ClassPathResource newDocumentTextDataClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_TEXT.proto");
        storageService.storeObject(TenantContext.getTenantId(), newDocumentTextDataStorageId, newDocumentTextDataClassPath.getInputStream());

        DocumentStructureProto.DocumentStructure documentStructure = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                    newDocumentStructureStorageId,
                                                                                                    DocumentStructureProto.DocumentStructure.parser());

        DocumentPageProto.AllDocumentPages allDocumentPages = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                             newDocumentPagesStorageId,
                                                                                             DocumentPageProto.AllDocumentPages.parser());

        DocumentTextDataProto.AllDocumentTextData allDocumentTextData = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                       newDocumentTextDataStorageId,
                                                                                                       DocumentTextDataProto.AllDocumentTextData.parser());

        DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                                   newDocumentPositionDataStorageId,
                                                                                                                   DocumentPositionDataProto.AllDocumentPositionData.parser());

    }


    @Test
    @SneakyThrows
    @Disabled
    public void testDocumentDataFilesConversionAndSaveToTempDir() {

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        String dossierId = dossier.getId();
        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        String fileId = file.getId();

        String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE.name(), ".json");
        String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES.name(), ".json");
        String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION.name(), ".json");
        String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT.name(), ".json");

        String newDocumentStructureStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
        String newDocumentPagesStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
        String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
        String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);

        ClassPathResource oldDocumentStructureClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_STRUCTURE.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentStructureStorageId, oldDocumentStructureClassPath.getInputStream());
        ClassPathResource oldDocumentPagesClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_PAGES.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentPagesStorageId, oldDocumentPagesClassPath.getInputStream());
        ClassPathResource oldDocumentPositionDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_POSITION.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId, oldDocumentPositionDataClassPath.getInputStream());
        ClassPathResource oldDocumentTextDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_TEXT.json");
        storageService.storeObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId, oldDocumentTextDataClassPath.getInputStream());

        when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));

        documentDataFilesConversionJob.execute(null);

        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_STRUCTURE));
        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_TEXT));
        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_PAGES));
        Assertions.assertTrue(fileManagementStorageService.objectExists(dossierId, fileId, FileType.DOCUMENT_POSITION));

        DocumentStructureProto.DocumentStructure documentStructure = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                    newDocumentStructureStorageId,
                                                                                                    DocumentStructureProto.DocumentStructure.parser());
        DocumentPageProto.AllDocumentPages allDocumentPages = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                             newDocumentPagesStorageId,
                                                                                             DocumentPageProto.AllDocumentPages.parser());
        DocumentTextDataProto.AllDocumentTextData allDocumentTextData = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                       newDocumentTextDataStorageId,
                                                                                                       DocumentTextDataProto.AllDocumentTextData.parser());
        DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                                   newDocumentPositionDataStorageId,
                                                                                                                   DocumentPositionDataProto.AllDocumentPositionData.parser());

        String tempDir = "/tmp";

        Path documentStructurePath = Path.of(tempDir, "new.DOCUMENT_STRUCTURE.proto");
        Files.write(documentStructurePath, documentStructure.toByteArray());
        Path documentPagesPath = Path.of(tempDir, "new.DOCUMENT_PAGES.proto");
        Files.write(documentPagesPath, allDocumentPages.toByteArray());
        Path documentTextPath = Path.of(tempDir, "new.DOCUMENT_TEXT.proto");
        Files.write(documentTextPath, allDocumentTextData.toByteArray());
        Path documentPositionPath = Path.of(tempDir, "new.DOCUMENT_POSITION.proto");
        Files.write(documentPositionPath, allDocumentPositionData.toByteArray());

        Assertions.assertTrue(Files.exists(documentStructurePath));
        Assertions.assertTrue(Files.exists(documentPagesPath));
        Assertions.assertTrue(Files.exists(documentTextPath));
        Assertions.assertTrue(Files.exists(documentPositionPath));
    }


    @Test
    @Disabled
    public void testParseProtoFiles() throws Exception {

        String tempDir = "/tmp";

        Path documentStructurePath = Path.of(tempDir, "new.DOCUMENT_STRUCTURE.proto");
        Path documentPagesPath = Path.of(tempDir, "new.DOCUMENT_PAGES.proto");
        Path documentTextPath = Path.of(tempDir, "new.DOCUMENT_TEXT.proto");
        Path documentPositionPath = Path.of(tempDir, "new.DOCUMENT_POSITION.proto");

        byte[] documentStructureBytes = Files.readAllBytes(documentStructurePath);
        byte[] documentPagesBytes = Files.readAllBytes(documentPagesPath);
        byte[] documentTextBytes = Files.readAllBytes(documentTextPath);
        byte[] documentPositionBytes = Files.readAllBytes(documentPositionPath);

        DocumentStructureProto.DocumentStructure documentStructure = DocumentStructureProto.DocumentStructure.parseFrom(documentStructureBytes);
        DocumentPageProto.AllDocumentPages allDocumentPages = DocumentPageProto.AllDocumentPages.parseFrom(documentPagesBytes);
        DocumentTextDataProto.AllDocumentTextData allDocumentTextData = DocumentTextDataProto.AllDocumentTextData.parseFrom(documentTextBytes);
        DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = DocumentPositionDataProto.AllDocumentPositionData.parseFrom(documentPositionBytes);
    }


    @Test
    public void testUserNotificationsWhenUnAssigned() { //RED-9588

        var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
        var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
        var file = fileTesterAndProvider.testAndProvideFile(dossier);
        var file2 = fileTesterAndProvider.testAndProvideFile(dossier, "file2");

        var currentUserId = userProvider.getUserId();
        var user1 = userProvider.getAltUserId();
        var user2 = userProvider.getMemberUserId();
        assertThat(currentUserId).isNotEqualTo(user1);
        assertThat(currentUserId).isNotEqualTo(user2);
        assertThat(user1).isNotEqualTo(user2);

        DossierRequest cru = new DossierRequest();
        cru.setDossierTemplateId(dossierTemplate.getId());
        BeanUtils.copyProperties(dossier, cru);
        cru.getMemberIds().add(user1);
        cru.getMemberIds().add(user2);
        dossierClient.createDossierOrUpdateDossier(cru);

        fileClient.setStatusUnderReviewForList(dossier.getId(), List.of(file.getId(), file2.getId()), user1);
        var fileStatus1 = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertEquals(WorkflowStatus.UNDER_REVIEW, fileStatus1.getWorkflowStatus());
        assertEquals(user1, fileStatus1.getAssignee());

        fileClient.setStatusNewForList(dossier.getId(), List.of(file.getId(), file2.getId()));
        fileStatus1 = fileClient.getFileStatus(dossier.getId(), file.getId());
        assertEquals(WorkflowStatus.NEW, fileStatus1.getWorkflowStatus());
        assertNull(fileStatus1.getAssignee());

        verify(notificationPersistenceService, times(5)).insertNotification(captor.capture());

        Assertions.assertEquals(captor.getValue().getUserId(), user1);
        Assertions.assertEquals(captor.getValue().getNotificationType(), NotificationType.UNASSIGNED_FROM_FILE.name());
    }

}
