From e6e0dd27dd9c8c2782cc55a7c6b76856f234d762 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Tue, 7 Jun 2022 12:17:14 +0300 Subject: [PATCH 1/2] RED-3842 filesize --- .../dossier/file/AddFileRequest.java | 1 - .../dossier/file/FileModel.java | 1 + .../processor/entity/dossier/FileEntity.java | 3 + .../FileStatusPersistenceService.java | 5 +- .../repository/FileRepository.java | 12 +++- .../migrations/FileSizeMigration8.java | 58 +++++++++++++++++++ .../FileStatusProcessingUpdateService.java | 5 +- .../v1/server/service/FileStatusService.java | 4 +- .../31-add-file-size-column.changelog.yaml | 12 ++++ persistence-service-v1/pom.xml | 2 +- 10 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/FileSizeMigration8.java create mode 100644 persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/31-add-file-size-column.changelog.yaml diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java index 4c00841b6..811211772 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java @@ -14,7 +14,6 @@ public class AddFileRequest { private String fileId; @NonNull private String dossierId; - private String uploader; } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java index 865b3176d..af0bbb66c 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java @@ -25,6 +25,7 @@ public class FileModel { private ProcessingStatus processingStatus; private WorkflowStatus workflowStatus; private int numberOfPages; + private long fileSize; private OffsetDateTime added; private OffsetDateTime lastUpdated; private OffsetDateTime deleted; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 03ca040e8..2c0977d74 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -55,6 +55,9 @@ public class FileEntity { @Column private int numberOfPages; + @Column + private long fileSize; + @Column private OffsetDateTime added; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 20b1b4c42..b529625ed 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -55,12 +55,13 @@ public class FileStatusPersistenceService { @Transactional - public void updateProcessingStatusPreprocessed(String fileId, boolean hasHighlights) { + public void updateProcessingStatusPreprocessed(String fileId, boolean hasHighlights, long fileSize) { if (isFileDeleted(fileId)) { return; } - fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights); + fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, + OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 08b852dca..90c539dab 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -58,9 +58,15 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) - @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.hasHighlights = :hasHighlights " + - "where f.id = :fileId") - void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights); + @Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," + + " f.hasHighlights = :hasHighlights, f.fileSize = :fileSize " + + " where f.id = :fileId") + void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize); + + + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.fileSize = :fileSize where f.id = :fileId") + void updateFileSize(String fileId, long fileSize); @Modifying(clearAutomatically = true) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/FileSizeMigration8.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/FileSizeMigration8.java new file mode 100644 index 000000000..92cb54515 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/FileSizeMigration8.java @@ -0,0 +1,58 @@ +package com.iqser.red.service.peristence.v1.server.migration.migrations; + +import com.iqser.red.service.peristence.v1.server.migration.Migration; +import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Slf4j +@Setter +@Service +public class FileSizeMigration8 extends Migration { + + private static final String NAME = "Update file size"; + private static final long VERSION = 8; + + @Autowired + private FileRepository fileRepository; + + @Autowired + private FileManagementStorageService fileManagementStorageService; + + + public FileSizeMigration8() { + + super(NAME, VERSION); + } + + + @Override + protected void migrate() { + + var allFiles = fileRepository.findAll(); + + allFiles.forEach(file -> { + // not hard deleted + if (file.getHardDeletedTime() == null) { + try { + var originFile = fileManagementStorageService.getStoredObjectBytes(file.getDossierId(), file.getId(), FileType.ORIGIN); + fileRepository.updateFileSize(file.getId(), originFile.length); + } catch (Exception e) { + log.warn("Failed to load file bytes for file: {} in dossier {} ", file.getId(), file.getDossierId()); + } + } + }); + + } + + +} + + + + + diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 8580d4dad..53798af8e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -72,10 +72,9 @@ public class FileStatusProcessingUpdateService { } - public void preprocessingSuccessful(String dossierId, String fileId, - UntouchedDocumentResponse untouchedDocumentResponse){ + public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse){ - fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights()); + fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(),untouchedDocumentResponse.getFileSize()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 62e66ca29..a6b300dae 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -117,9 +117,9 @@ public class FileStatusService { } - public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights) { + public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) { - fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights); + fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights,fileSize); addToAnalysisQueue(dossierId, fileId, false, Set.of()); if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/31-add-file-size-column.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/31-add-file-size-column.changelog.yaml new file mode 100644 index 000000000..b0ea8a23a --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/31-add-file-size-column.changelog.yaml @@ -0,0 +1,12 @@ +databaseChangeLog: + - changeSet: + id: add-update-file-size-column + author: timo + changes: + - addColumn: + columns: + - column: + name: file_size + type: BIGINT + tableName: file + diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index a1c60723d..b0bc933fa 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -27,7 +27,7 @@ 3.114.0 2.36.0 - 3.83.0 + 3.96.0 3.47.0 From 5171b039d864da5ae4445ce08e234e8ed26c465a Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Tue, 7 Jun 2022 12:23:25 +0300 Subject: [PATCH 2/2] RED-3842 filesize --- .../migration/migrations/MigrateHighlights3.java | 11 +++++------ .../db/changelog/db.changelog-master.yaml | 2 ++ .../AbstractPersistenceServerServiceTest.java | 14 -------------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java index 5a90c2a2f..0039ffb27 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java @@ -1,8 +1,8 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; +import com.iqser.red.service.peristence.v1.server.service.FileStatusService; import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; @@ -27,10 +27,10 @@ public class MigrateHighlights3 extends Migration { private FileStatusPersistenceService fileStatusPersistenceService; @Autowired - private PDFTronRedactionClient pdfTronRedactionClient; + private FileManagementStorageService fileManagementStorageService; @Autowired - private FileManagementStorageService fileManagementStorageService; + private FileStatusService fileStatusService; public MigrateHighlights3() { @@ -63,10 +63,9 @@ public class MigrateHighlights3 extends Migration { fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN)); } - var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder() - .fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build()); - fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights()); + fileStatusService.addToPreprocessingQueue(file.getId(), file.getDossierId(), file.getFilename()); + } else { log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 37055c1ce..9e28ec0d8 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -71,3 +71,5 @@ databaseChangeLog: file: db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml - include: file: db/changelog/sql/30-change-bigint-to-serial.sql + - include: + file: db/changelog/31-add-file-size-column.changelog.yaml diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index 85eb340dd..366821724 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -148,20 +148,6 @@ public abstract class AbstractPersistenceServerServiceTest { when(amqpAdmin.getQueueInfo(Mockito.any())).thenReturn(null); - doAnswer(answer -> { - - Object[] args = answer.getArguments(); - DocumentRequest d = (DocumentRequest) args[0]; - - var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.UNTOUCHED); - - var untouched = storageService.getObject(untouchedObjectId); - storageService.storeObject(StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN), IOUtils.toByteArray(untouched.getInputStream())); - return UntouchedDocumentResponse.builder().build(); - - }).when(pdfTronRedactionClient).processUntouchedDocument(any()); - - when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((answer) -> { Object[] args = answer.getArguments(); DocumentRequest d = (DocumentRequest) args[0];