Merge branch 'feature/RED-10347' into 'master'

RED-10347: Last download time field for approved files

Closes RED-10347

See merge request redactmanager/persistence-service!823
This commit is contained in:
Maverick Studer 2024-11-28 10:02:59 +01:00
commit d3f0d1bc87
14 changed files with 259 additions and 35 deletions

View File

@ -9,6 +9,7 @@ import java.util.Set;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -22,8 +23,10 @@ import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.ForeignKey;
import jakarta.persistence.Id;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.AllArgsConstructor;
@ -218,6 +221,11 @@ public class FileEntity {
@Column
private boolean protobufMigrationDone;
@ManyToOne(fetch = FetchType.EAGER)
@JoinColumn(name = "last_download", referencedColumnName = "storage_id", foreignKey = @ForeignKey(name = "fk_file_last_download"))
private DownloadStatusEntity lastDownload;
public OffsetDateTime getLastOCRTime() {
return this.ocrStartTime;

View File

@ -43,6 +43,7 @@ import lombok.experimental.FieldDefaults;
public class DownloadStatusEntity {
@Id
@Column(name = "storage_id")
String storageId;
@Column
String uuid;

View File

@ -125,7 +125,7 @@ public class FileStatusManagementService {
if (userId != null) {
assignee = userId;
}
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.NEW);
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_REVIEW : WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
fileStatusService.setAssignee(fileId, assignee);
indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2);
}
@ -139,7 +139,7 @@ public class FileStatusManagementService {
assignee = approverId;
}
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_APPROVAL : WorkflowStatus.NEW);
fileStatusService.setStatusSuccessful(fileId, assignee != null ? WorkflowStatus.UNDER_APPROVAL : WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
fileStatusService.setAssignee(fileId, approverId);
indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, fileId, 2);
}
@ -169,7 +169,7 @@ public class FileStatusManagementService {
throw new BadRequestException("Allowed transition not possible from: " + fileStatus.getWorkflowStatus() + " to status NEW");
}
fileStatusService.setAssignee(fileId, null);
fileStatusService.setStatusSuccessful(fileId, WorkflowStatus.NEW);
fileStatusService.setStatusSuccessful(fileId, WorkflowStatus.NEW, fileStatus.getWorkflowStatus());
}

View File

@ -69,6 +69,7 @@ public class FileStatusMapper {
.fileSize(status.getFileSize())
.fileErrorInfo(status.getFileErrorInfo())
.componentMappingVersions(status.getComponentMappingVersions())
.lastDownload(status.getLastDownloadDate())
.build();
}

View File

@ -816,9 +816,13 @@ public class FileStatusService {
}
public void setStatusSuccessful(String fileId, WorkflowStatus workflowStatus) {
public void setStatusSuccessful(String fileId, WorkflowStatus newWorkflowStatus, WorkflowStatus oldWorkflowStatus) {
fileStatusPersistenceService.updateWorkflowStatus(fileId, workflowStatus, false);
fileStatusPersistenceService.updateWorkflowStatus(fileId, newWorkflowStatus, false);
if(oldWorkflowStatus == WorkflowStatus.APPROVED && newWorkflowStatus != WorkflowStatus.APPROVED) {
fileStatusPersistenceService.clearLastDownload(fileId);
}
}

View File

@ -34,19 +34,47 @@ public class DownloadStatusPersistenceService {
// use this to create a status for export dossier template.
public void createStatus(String userId, String storageId, String filename, String mimeType) {
this.createStatus(userId, storageId, null, filename, mimeType, null, null, null, null);
this.saveDownloadStatus(userId, storageId, null, filename, mimeType, null, null, null, null);
}
@Transactional
public DownloadStatusEntity createStatus(String userId,
String storageId,
DossierEntity dossier,
String filename,
String mimeType,
List<String> fileIds,
Set<DownloadFileType> downloadFileTypes,
List<String> reportTemplateIds,
String redactionPreviewColor) {
String storageId,
DossierEntity dossier,
String filename,
String mimeType,
List<String> fileIds,
Set<DownloadFileType> downloadFileTypes,
List<String> reportTemplateIds,
String redactionPreviewColor) {
DownloadStatusEntity savedDownloadStatus = saveDownloadStatus(userId,
storageId,
dossier,
filename,
mimeType,
fileIds,
downloadFileTypes,
reportTemplateIds,
redactionPreviewColor);
if (fileIds != null && !fileIds.isEmpty()) {
fileRepository.updateLastDownloadForApprovedFiles(fileIds, savedDownloadStatus);
}
return savedDownloadStatus;
}
private DownloadStatusEntity saveDownloadStatus(String userId,
String storageId,
DossierEntity dossier,
String filename,
String mimeType,
List<String> fileIds,
Set<DownloadFileType> downloadFileTypes,
List<String> reportTemplateIds,
String redactionPreviewColor) {
DownloadStatusEntity downloadStatus = new DownloadStatusEntity();
downloadStatus.setUserId(userId);
@ -62,7 +90,7 @@ public class DownloadStatusPersistenceService {
downloadStatus.setRedactionPreviewColor(redactionPreviewColor);
downloadStatus.setStatus(DownloadStatusValue.QUEUED);
return downloadStatusRepository.save(downloadStatus);
return downloadStatusRepository.saveAndFlush(downloadStatus);
}
@ -134,7 +162,8 @@ public class DownloadStatusPersistenceService {
@Transactional
public DownloadStatusEntity getStatusesByUuid(String uuid) {
return downloadStatusRepository.findByUuid(uuid).orElseThrow(() -> new NotFoundException(String.format("DownloadStatus not found for uuid: %s", uuid)));
return downloadStatusRepository.findByUuid(uuid)
.orElseThrow(() -> new NotFoundException(String.format("DownloadStatus not found for uuid: %s", uuid)));
}

View File

@ -3,7 +3,6 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@ -754,4 +753,11 @@ public class FileStatusPersistenceService {
return fileRepository.findAllByDossierId(dossierId, includeDeleted);
}
@Transactional
public void clearLastDownload(String fileId) {
fileRepository.updateLastDownloadForFile(fileId, null);
}
}

View File

@ -12,6 +12,7 @@ import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierChangeProjection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileChangeProjection;
@ -142,8 +143,7 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.processingErrorCounter = :processingErrorCounter "
+ "where f.dossierId in (select fe.dossierId from FileEntity fe inner join DossierEntity d on d.id = fe.dossierId where d.dossierTemplateId = :dossierTemplateId) and f.processingStatus = 'ERROR'")
void updateErrorCounter(@Param("dossierTemplateId") String dossierTemplateId,
@Param("processingErrorCounter") int processingErrorCounter);
void updateErrorCounter(@Param("dossierTemplateId") String dossierTemplateId, @Param("processingErrorCounter") int processingErrorCounter);
@Modifying
@ -467,6 +467,16 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
List<String> findAllByDossierId(@Param("dossierId") String dossierId, @Param("includeDeleted") boolean includeDeleted);
@Modifying
@Query("UPDATE FileEntity f SET f.lastDownload = :lastDownload WHERE f.id IN :fileIds AND f.workflowStatus = 'APPROVED'")
void updateLastDownloadForApprovedFiles(@Param("fileIds") List<String> fileIds, @Param("lastDownload") DownloadStatusEntity lastDownload);
@Modifying
@Query("UPDATE FileEntity f SET f.lastDownload = :lastDownload WHERE f.id = :fileId")
void updateLastDownloadForFile(@Param("fileId") String fileId, @Param("lastDownload") DownloadStatusEntity lastDownload);
@Query("SELECT f FROM FileEntity f WHERE f.id in :fileIds AND f.dossierId = :dossierId")
List<FileEntity> findAllDossierIdAndIds(@Param("dossierId") String dossierId, @Param("fileIds") Set<String> fileIds);

View File

@ -20,6 +20,9 @@ public class FileModelMapper implements BiConsumer<FileEntity, FileModel> {
.forEach(fa -> fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue()));
fileModel.setFileErrorInfo(new FileErrorInfo(fileEntity.getErrorCause(), fileEntity.getErrorQueue(), fileEntity.getErrorService(), fileEntity.getErrorTimestamp(), fileEntity.getErrorCode()));
fileModel.setComponentMappingVersions(getComponentMappingVersions(fileEntity));
if (fileEntity.getLastDownload() != null) {
fileModel.setLastDownloadDate(fileEntity.getLastDownload().getCreationDate());
}
}

View File

@ -249,3 +249,5 @@ databaseChangeLog:
file: db/changelog/tenant/152-add-ai-fields-to-entity.yaml
- include:
file: db/changelog/tenant/153-custom-technical-name-change.yaml
- include:
file: db/changelog/tenant/154-add-last-download-to-file.yaml

View File

@ -0,0 +1,28 @@
databaseChangeLog:
- changeSet:
id: add-last-download-to-file
author: maverick
changes:
- addColumn:
tableName: file
columns:
- column:
name: last_download
type: VARCHAR(255)
constraints:
nullable: true
- changeSet:
id: add-fk-last-download
author: maverick
changes:
- addForeignKeyConstraint:
baseTableName: file
baseColumnNames: last_download
constraintName: fk_file_last_download
referencedTableName: download_status
referencedColumnNames: storage_id
onDelete: SET NULL
onUpdate: NO ACTION
deferrable: false
initiallyDeferred: false
validate: true

View File

@ -4,6 +4,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -57,6 +58,7 @@ import org.junit.jupiter.api.Test;
public class DownloadPreparationTest extends AbstractPersistenceServerServiceTest {
protected static final String USER_ID = "1";
@Autowired
DownloadReportMessageReceiver downloadReportMessageReceiver;
@ -87,8 +89,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
@Autowired
FileClient fileClient;
DossierWithSingleFile testData;
@Autowired
DownloadReadyJob downloadReadyJob;
@ -98,14 +98,13 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
@Autowired
DownloadCompressionMessageReceiver downloadCompressionMessageReceiver;
DossierWithSingleFile testData;
@BeforeEach
public void createTestData() {
testData = new DossierWithSingleFile();
}
@Test
@SneakyThrows
public void testReceiveDownloadPackage() {
@ -159,7 +158,7 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null); // Will be called by scheduler in prod.
var firstStatus = getFirstStatus();
assertThat(getFirstStatus().getStatus()).isEqualTo(DownloadStatusValue.COMPRESSING);
assertThat(firstStatus.getStatus()).isEqualTo(DownloadStatusValue.COMPRESSING);
downloadCompressionMessageReceiver.receive(DownloadJob.builder().storageId(firstStatus.getStorageId()).build());
firstStatus = getFirstStatus();
@ -169,6 +168,133 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
clearTenantContext();
}
@Test
@SneakyThrows
public void testLastDownloadSetForApprovedFile() {
// Arrange
testData.forwardFileToApprovedState();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Act
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
// Trigger the download processing job
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Assert
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNotNull();
}
@Test
@SneakyThrows
public void testLastDownloadNotSetForNonApprovedFile() {
// Arrange
testData.forwardFile();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Act
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
// Trigger the download processing job
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Assert
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNull();
}
@Test
@SneakyThrows
public void testLastDownloadResetWhenStatusChangedFromApprovedToDifferent() {
// Arrange
testData.forwardFileToApprovedState();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Prepare and process download to set 'last_download'
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Verify that 'last_download' is set
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNotNull();
// Change status from approved to reviewed
fileClient.setStatusUnderReview(testData.getDossierId(), testData.getFileId(), null);
// Assert that 'last_download' is reset
FileStatus updatedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(updatedFileStatus.getLastDownload()).isNull();
}
@Test
@SneakyThrows
public void testLastDownloadRemainsWhenStatusChangedFromApprovedToApproved() {
// Arrange
testData.forwardFileToApprovedState();
uploadMockReportTemplate(testData);
var availableTemplates = reportTemplateClient.getAvailableReportTemplates(testData.getDossierTemplateId());
assertThat(availableTemplates).isNotEmpty();
createDossierInService(testData, availableTemplates);
// Prepare and process download to set 'last_download'
downloadClient.prepareDownload(PrepareDownloadWithOptionRequest.builder()
.dossierId(testData.getDossierId())
.downloadFileTypes(Set.of(DownloadFileType.ORIGINAL))
.fileIds(Collections.singletonList(testData.file.getId()))
.redactionPreviewColor("#aaaaaa")
.build());
setupTenantContext();
when(this.tenantsClient.getTenants()).thenReturn(List.of(TenantResponse.builder().tenantId("redaction").details(Map.of("persistence-service-ready", true)).build()));
downloadReadyJob.execute(null);
clearTenantContext();
// Verify that 'last_download' is set
FileStatus persistedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(persistedFileStatus.getLastDownload()).isNotNull();
// Change status from approved to approved
fileClient.setStatusApproved(testData.getDossierId(), testData.getFileId(), true);
// Assert that 'last_download' remains set
FileStatus updatedFileStatus = fileClient.getFileStatus(testData.getDossierId(), testData.getFileId());
assertThat(updatedFileStatus.getLastDownload()).isNotNull();
}
private void createDossierInService(DossierWithSingleFile testData, List<ReportTemplate> availableTemplates) {
@ -189,18 +315,16 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
.build());
}
private void uploadMockReportTemplate(DossierWithSingleFile testData) {
var template = new MockMultipartFile("test.docx", "zzz".getBytes());
reportTemplateClient.uploadTemplate(template, testData.getDossierTemplateId(), true, true);
}
@SneakyThrows
private void addStoredFileInformationToStorage(FileStatus file, List<ReportTemplate> availableTemplates, String downloadId) {
var storedFileInformationstorageId = downloadId.substring(0, downloadId.length() - 3) + "/REPORT_INFO.json";
var storedFileInformationStorageId = downloadId.substring(0, downloadId.length() - 3) + "/REPORT_INFO.json";
String reportStorageId = "XYZ";
var sivList = new ArrayList<StoredFileInformation>();
@ -210,11 +334,10 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
siv.setTemplateId(availableTemplates.iterator().next().getTemplateId());
sivList.add(siv);
storageService.storeObject(TenantContext.getTenantId(), storedFileInformationstorageId, new ByteArrayInputStream(new ObjectMapper().writeValueAsBytes(sivList)));
storageService.storeObject(TenantContext.getTenantId(), storedFileInformationStorageId, new ByteArrayInputStream(new ObjectMapper().writeValueAsBytes(sivList)));
storageService.storeObject(TenantContext.getTenantId(), reportStorageId, new ByteArrayInputStream(new byte[]{1, 2, 3, 4}));
}
private DownloadStatus getFirstStatus() {
List<DownloadStatus> finalDownloadStatuses = downloadClient.getDownloadStatus().getDownloadStatus();
@ -232,21 +355,15 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
FileStatus file = fileTesterAndProvider.testAndProvideFile(dossier);
public String getDossierTemplateId() {
return dossierTemplate.getId();
}
public String getDossierId() {
return dossier.getId();
}
public String getFileId() {
return file.getFileId();
}
@ -260,6 +377,12 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
assertThatTestFileIsApproved();
}
public void forwardFile() {
fileTesterAndProvider.markFileAsProcessed(getDossierId(), getFileId());
assertThatTestFileIsNew();
}
private void assertThatTestFileIsApproved() {
@ -267,6 +390,12 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
assertThat(persistedFileStatus.getWorkflowStatus()).isEqualTo(WorkflowStatus.APPROVED);
}
private void assertThatTestFileIsNew() {
var persistedFileStatus = fileClient.getFileStatus(getDossierId(), file.getId());
assertThat(persistedFileStatus.getWorkflowStatus()).isEqualTo(WorkflowStatus.NEW);
}
}
}

View File

@ -154,6 +154,8 @@ public class FileStatus {
private FileErrorInfo fileErrorInfo;
@Schema(description = "Shows which version of each mapping the last analysis has been performed")
private Map<String, Integer> componentMappingVersions;
@Schema(description = "Last time the approved file was downloaded")
private OffsetDateTime lastDownload;
@Schema(description = "Shows if this file has been OCRed by us. Last Time of OCR.")
public OffsetDateTime getLastOCRTime() {

View File

@ -78,6 +78,7 @@ public class FileModel {
private boolean hasHighlights;
private FileErrorInfo fileErrorInfo;
private Map<String, Integer> componentMappingVersions = new HashMap<>();
private OffsetDateTime lastDownloadDate;
public long getFileSize() {