Pull request #434: RED-3842
Merge in RED/persistence-service from RED-3842 to master * commit '5171b039d864da5ae4445ce08e234e8ed26c465a': RED-3842 filesize RED-3842 filesize
This commit is contained in:
Timo Bejan
commit
c3c34e9b4f
@ -14,7 +14,6 @@ public class AddFileRequest {
|
||||
private String fileId;
|
||||
@NonNull
|
||||
private String dossierId;
|
||||
|
||||
private String uploader;
|
||||
|
||||
}
|
||||
|
||||
@ -25,6 +25,7 @@ public class FileModel {
|
||||
private ProcessingStatus processingStatus;
|
||||
private WorkflowStatus workflowStatus;
|
||||
private int numberOfPages;
|
||||
private long fileSize;
|
||||
private OffsetDateTime added;
|
||||
private OffsetDateTime lastUpdated;
|
||||
private OffsetDateTime deleted;
|
||||
|
||||
@ -55,6 +55,9 @@ public class FileEntity {
|
||||
@Column
|
||||
private int numberOfPages;
|
||||
|
||||
@Column
|
||||
private long fileSize;
|
||||
|
||||
@Column
|
||||
private OffsetDateTime added;
|
||||
|
||||
|
||||
@ -55,12 +55,13 @@ public class FileStatusPersistenceService {
|
||||
|
||||
|
||||
@Transactional
|
||||
public void updateProcessingStatusPreprocessed(String fileId, boolean hasHighlights) {
|
||||
public void updateProcessingStatusPreprocessed(String fileId, boolean hasHighlights, long fileSize) {
|
||||
|
||||
if (isFileDeleted(fileId)) {
|
||||
return;
|
||||
}
|
||||
fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
|
||||
fileRepository.updateProcessingStatus(fileId, ProcessingStatus.PRE_PROCESSED,
|
||||
OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights, fileSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -58,9 +58,15 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, f.hasHighlights = :hasHighlights " +
|
||||
"where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights);
|
||||
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
|
||||
" f.hasHighlights = :hasHighlights, f.fileSize = :fileSize " +
|
||||
" where f.id = :fileId")
|
||||
void updateProcessingStatus(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, boolean hasHighlights, long fileSize);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
@Query("update FileEntity f set f.fileSize = :fileSize where f.id = :fileId")
|
||||
void updateFileSize(String fileId, long fileSize);
|
||||
|
||||
|
||||
@Modifying(clearAutomatically = true)
|
||||
|
||||
@ -0,0 +1,58 @@
|
||||
package com.iqser.red.service.peristence.v1.server.migration.migrations;
|
||||
|
||||
import com.iqser.red.service.peristence.v1.server.migration.Migration;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Slf4j
|
||||
@Setter
|
||||
@Service
|
||||
public class FileSizeMigration8 extends Migration {
|
||||
|
||||
private static final String NAME = "Update file size";
|
||||
private static final long VERSION = 8;
|
||||
|
||||
@Autowired
|
||||
private FileRepository fileRepository;
|
||||
|
||||
@Autowired
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
|
||||
public FileSizeMigration8() {
|
||||
|
||||
super(NAME, VERSION);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void migrate() {
|
||||
|
||||
var allFiles = fileRepository.findAll();
|
||||
|
||||
allFiles.forEach(file -> {
|
||||
// not hard deleted
|
||||
if (file.getHardDeletedTime() == null) {
|
||||
try {
|
||||
var originFile = fileManagementStorageService.getStoredObjectBytes(file.getDossierId(), file.getId(), FileType.ORIGIN);
|
||||
fileRepository.updateFileSize(file.getId(), originFile.length);
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to load file bytes for file: {} in dossier {} ", file.getId(), file.getDossierId());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
package com.iqser.red.service.peristence.v1.server.migration.migrations;
|
||||
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
|
||||
import com.iqser.red.service.peristence.v1.server.migration.Migration;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
|
||||
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
|
||||
@ -27,10 +27,10 @@ public class MigrateHighlights3 extends Migration {
|
||||
private FileStatusPersistenceService fileStatusPersistenceService;
|
||||
|
||||
@Autowired
|
||||
private PDFTronRedactionClient pdfTronRedactionClient;
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
|
||||
@Autowired
|
||||
private FileManagementStorageService fileManagementStorageService;
|
||||
private FileStatusService fileStatusService;
|
||||
|
||||
public MigrateHighlights3() {
|
||||
|
||||
@ -63,10 +63,9 @@ public class MigrateHighlights3 extends Migration {
|
||||
fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
|
||||
|
||||
}
|
||||
var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder()
|
||||
.fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build());
|
||||
|
||||
fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights());
|
||||
fileStatusService.addToPreprocessingQueue(file.getId(), file.getDossierId(), file.getFilename());
|
||||
|
||||
} else {
|
||||
log.warn("Invalid file: {} in dossier: {}. File Data ( PDF ) does not exist", file.getId(), file.getDossierId());
|
||||
}
|
||||
|
||||
@ -72,10 +72,9 @@ public class FileStatusProcessingUpdateService {
|
||||
}
|
||||
|
||||
|
||||
public void preprocessingSuccessful(String dossierId, String fileId,
|
||||
UntouchedDocumentResponse untouchedDocumentResponse){
|
||||
public void preprocessingSuccessful(String dossierId, String fileId, UntouchedDocumentResponse untouchedDocumentResponse){
|
||||
|
||||
fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights());
|
||||
fileStatusService.updateProcessingStatusPreprocessed(dossierId, fileId, untouchedDocumentResponse.isHasHighlights(),untouchedDocumentResponse.getFileSize());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -117,9 +117,9 @@ public class FileStatusService {
|
||||
}
|
||||
|
||||
|
||||
public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights) {
|
||||
public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) {
|
||||
|
||||
fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights);
|
||||
fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights,fileSize);
|
||||
addToAnalysisQueue(dossierId, fileId, false, Set.of());
|
||||
|
||||
if (fileManagementServiceSettings.isPdf2ImageServiceEnabled()) {
|
||||
|
||||
@ -0,0 +1,12 @@
|
||||
databaseChangeLog:
|
||||
- changeSet:
|
||||
id: add-update-file-size-column
|
||||
author: timo
|
||||
changes:
|
||||
- addColumn:
|
||||
columns:
|
||||
- column:
|
||||
name: file_size
|
||||
type: BIGINT
|
||||
tableName: file
|
||||
|
||||
@ -71,3 +71,5 @@ databaseChangeLog:
|
||||
file: db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml
|
||||
- include:
|
||||
file: db/changelog/sql/30-change-bigint-to-serial.sql
|
||||
- include:
|
||||
file: db/changelog/31-add-file-size-column.changelog.yaml
|
||||
|
||||
@ -148,20 +148,6 @@ public abstract class AbstractPersistenceServerServiceTest {
|
||||
|
||||
when(amqpAdmin.getQueueInfo(Mockito.any())).thenReturn(null);
|
||||
|
||||
doAnswer(answer -> {
|
||||
|
||||
Object[] args = answer.getArguments();
|
||||
DocumentRequest d = (DocumentRequest) args[0];
|
||||
|
||||
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.UNTOUCHED);
|
||||
|
||||
var untouched = storageService.getObject(untouchedObjectId);
|
||||
storageService.storeObject(StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN), IOUtils.toByteArray(untouched.getInputStream()));
|
||||
return UntouchedDocumentResponse.builder().build();
|
||||
|
||||
}).when(pdfTronRedactionClient).processUntouchedDocument(any());
|
||||
|
||||
|
||||
when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((answer) -> {
|
||||
Object[] args = answer.getArguments();
|
||||
DocumentRequest d = (DocumentRequest) args[0];
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
<properties>
|
||||
<redaction-service.version>3.114.0</redaction-service.version>
|
||||
<search-service.version>2.36.0</search-service.version>
|
||||
<pdftron-redaction-service.version>3.83.0</pdftron-redaction-service.version>
|
||||
<pdftron-redaction-service.version>3.96.0</pdftron-redaction-service.version>
|
||||
<redaction-report-service.version>3.47.0</redaction-report-service.version>
|
||||
</properties>
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user