Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-8175' into 'master'

Browse Source 
RED-8175: Use only one thread for @Async deletion of files that are already...

Closes RED-8175

See merge request redactmanager/persistence-service!304
This commit is contained in:
Dominique Eifländer 2024-01-11 12:47:50 +01:00
commit e2b64b566b
1 changed files with 25 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/download/DownloadPreparationService.java
View File

@ -30,18 +30,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.download.Do
import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import jakarta.transaction.Transactional; import jakarta.transaction.Transactional;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j @Slf4j
@Service @Service
@ -69,7 +72,6 @@ public class DownloadPreparationService {
var downloadStatus = downloadStatusPersistenceService.getStatus(downloadId); var downloadStatus = downloadStatusPersistenceService.getStatus(downloadId);
RedactionMessage.RedactionMessageBuilder messageBuilder = this.generateGeneralRedactionMessage(downloadId, downloadStatus); RedactionMessage.RedactionMessageBuilder messageBuilder = this.generateGeneralRedactionMessage(downloadId, downloadStatus);
downloadStatus.getFiles().forEach(fileEntity -> { downloadStatus.getFiles().forEach(fileEntity -> {
RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).build(); RedactionMessage message = messageBuilder.fileId(fileEntity.getId()).unapprovedFile(fileEntity.getWorkflowStatus() != WorkflowStatus.APPROVED).build();
log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId()); log.info("Sending redaction request for downloadId:{} fileId:{} to pdftron-redaction-queue", downloadId, fileEntity.getId());
@ -77,6 +79,7 @@ public class DownloadPreparationService {
}); });
} }
private RedactionMessage.RedactionMessageBuilder generateGeneralRedactionMessage(String downloadId, DownloadStatusEntity downloadStatus) { private RedactionMessage.RedactionMessageBuilder generateGeneralRedactionMessage(String downloadId, DownloadStatusEntity downloadStatus) {
var dossier = downloadStatus.getDossier(); var dossier = downloadStatus.getDossier();
@ -94,6 +97,7 @@ public class DownloadPreparationService {
return buildRedactionMessage(downloadId, downloadStatus, dossier, dossierTemplate, previewColor, appliedRedactionColor); return buildRedactionMessage(downloadId, downloadStatus, dossier, dossierTemplate, previewColor, appliedRedactionColor);
} }
private RedactionMessage.RedactionMessageBuilder buildRedactionMessage(String downloadId, private RedactionMessage.RedactionMessageBuilder buildRedactionMessage(String downloadId,
DownloadStatusEntity downloadStatus, DownloadStatusEntity downloadStatus,
DossierEntity dossier, DossierEntity dossier,
@ -130,6 +134,7 @@ public class DownloadPreparationService {
return result; return result;
} }
@Transactional @Transactional
public void markFileAsProcessed(RedactionResultMessage redactionResultMessage) { public void markFileAsProcessed(RedactionResultMessage redactionResultMessage) {
@ -201,12 +206,9 @@ public class DownloadPreparationService {
.build()); .build());
} }
downloadReportCleanupService.deleteTmpReportFiles(storedFileInformations.stream().map(StoredFileInformation::getStorageId).collect(Collectors.toSet())); downloadReportCleanupService.deleteTmpReportFiles(Stream.concat(storedFileInformations.stream().map(StoredFileInformation::getStorageId), //
redactionFileResults.forEach(redactionFileResult -> downloadReportCleanupService.deleteTmpReportFiles(redactionFileResult.getDetails() redactionFileResults.stream().map(DownloadRedactionFileStatusEntity::getDetails).flatMap(List::stream).map(RedactionResultDetail::getStorageId)) //
.stream() .collect(Collectors.toSet()));
.map(RedactionResultDetail::getStorageId)
.collect(Collectors.toSet())));
} }
@ -216,7 +218,9 @@ public class DownloadPreparationService {
} }
private void generateAndAddFiles(DownloadStatusEntity downloadStatus, List<DownloadRedactionFileStatusEntity> redactionFileResults, FileSystemBackedArchiver fileSystemBackedArchiver) { private void generateAndAddFiles(DownloadStatusEntity downloadStatus,
List<DownloadRedactionFileStatusEntity> redactionFileResults,
FileSystemBackedArchiver fileSystemBackedArchiver) {
int i = 1; int i = 1;
long fileGenerationStart = System.currentTimeMillis(); long fileGenerationStart = System.currentTimeMillis();
@ -251,10 +255,17 @@ public class DownloadPreparationService {
} }
} }
log.info("Successfully added file {}/{} for downloadId {}, took {}", i, downloadStatus.getFiles().size(), downloadStatus.getStorageId(), System.currentTimeMillis() - start); log.info("Successfully added file {}/{} for downloadId {}, took {}",
i,
downloadStatus.getFiles().size(),
downloadStatus.getStorageId(),
System.currentTimeMillis() - start);
i++; i++;
} }
log.info("Successfully added {} files for downloadId {}, took {}", downloadStatus.getFiles().stream().map(FileEntity::getId).toList(), downloadStatus.getStorageId(), System.currentTimeMillis() - fileGenerationStart); log.info("Successfully added {} files for downloadId {}, took {}",
downloadStatus.getFiles().stream().map(FileEntity::getId).toList(),
downloadStatus.getStorageId(),
System.currentTimeMillis() - fileGenerationStart);
} }
@ -293,7 +304,10 @@ public class DownloadPreparationService {
} }
private void addReports(String downloadId, DownloadStatusEntity downloadStatus, List<StoredFileInformation> storedFileInformations, FileSystemBackedArchiver fileSystemBackedArchiver) { private void addReports(String downloadId,
DownloadStatusEntity downloadStatus,
List<StoredFileInformation> storedFileInformations,
FileSystemBackedArchiver fileSystemBackedArchiver) {
long addReportsStart = System.currentTimeMillis(); long addReportsStart = System.currentTimeMillis();