Pull request #475: RED-4527: Store information of generated reports in storage

Merge in RED/persistence-service from RED-4527 to master * commit 'a775d52cad4890b3d1be3ff942617a8dc8d6bf42': RED-4527: Store information of generated reports in storage
2022-07-07 11:31:40 +02:00 · 2022-07-07 11:31:40 +02:00 · ea6d359d13
commit ea6d359d13
parent b83ad16ee7 a775d52cad
6 changed files with 42 additions and 32 deletions
--- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java
+++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java
@ -23,10 +23,8 @@ import org.hibernate.annotations.FetchMode;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStoredFileInformationConverter;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
 import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
-import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@ -74,8 +72,4 @@ public class DownloadStatusEntity {
    @Convert(converter = JSONDownloadFileTypeConverter.class)
    Set<DownloadFileType> downloadFileTypes = new HashSet<>();

-    @Builder.Default
-    @Column(columnDefinition = "text", name = "generated_reports_information")
-    @Convert(converter = JSONStoredFileInformationConverter.class)
-    Set<StoredFileInformation> generatedReportsInformation = new HashSet<>();
 }
--- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java
@ -2,7 +2,6 @@ package com.iqser.red.service.peristence.v1.server.service.download;

 import java.io.IOException;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@ -15,6 +14,7 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;

 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
@ -41,6 +41,7 @@ import com.iqser.red.storage.commons.service.StorageService;

 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;

@ -66,8 +67,6 @@ public class DownloadPreparationService {
    public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException {

        DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
-        downloadStatus.setGeneratedReportsInformation(new HashSet<>(reportResultMessage.getStoredFileInformation()));
-        downloadStatusPersistenceService.updateStatusEntity(downloadStatus);

        DossierEntity dossier = downloadStatus.getDossier();

@ -112,10 +111,12 @@ public class DownloadPreparationService {

        DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());

+        var storedFileInformations = getStoredFileInformation(reportResultMessage.getDownloadId());
+
        try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {

            generateAndAddFiles(downloadStatus, reportResultMessage, fileSystemBackedArchiver);
-            addReports(downloadStatus, fileSystemBackedArchiver);
+            addReports(reportResultMessage.getDownloadId(), storedFileInformations, fileSystemBackedArchiver);
            storeZipFile(downloadStatus, fileSystemBackedArchiver);

            downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
@ -130,7 +131,7 @@ public class DownloadPreparationService {
                    .build());
        }

-        downloadReportCleanupService.deleteTmpReportFiles(downloadStatus.getGeneratedReportsInformation()
+        downloadReportCleanupService.deleteTmpReportFiles(storedFileInformations
                .stream()
                .map(StoredFileInformation::getStorageId)
                .collect(Collectors.toSet()));
@ -210,11 +211,11 @@ public class DownloadPreparationService {
    }


-    private void addReports(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
+    private void addReports(String downloadId, List<StoredFileInformation> storedFileInformations, FileSystemBackedArchiver fileSystemBackedArchiver) {

        long addReportsStart = System.currentTimeMillis();

-        for (StoredFileInformation storedFileInformation : downloadStatus.getGeneratedReportsInformation()) {
+        for (StoredFileInformation storedFileInformation : storedFileInformations) {

            FileEntity fileStatus = null;

@ -227,10 +228,17 @@ public class DownloadPreparationService {
            fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report));

        }
-        log.info("Successfully added {} reports for downloadId {}, took {}", downloadStatus.getGeneratedReportsInformation()
-                .size(), downloadStatus.getStorageId(), System.currentTimeMillis() - addReportsStart);
+        log.info("Successfully added {} reports for downloadId {}, took {}", storedFileInformations.size(), downloadId, System.currentTimeMillis() - addReportsStart);
    }

+    @SneakyThrows
+    private List<StoredFileInformation> getStoredFileInformation(String downloadId){
+
+        var storageId = downloadId.substring(0, downloadId.length() - 3) + "/REPORT_INFO.json";
+
+        return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() {
+        });
+    }

    private String createFileName(FileEntity fileStatus, ReportTemplateEntity reportTemplate) {

--- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/36-revert-reports-information-column.changelog.yaml
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/36-revert-reports-information-column.changelog.yaml
@ -0,0 +1,10 @@
+databaseChangeLog:
+  - changeSet:
+      id: remove-reports-information-column
+      author: dom
+      changes:
+        - removeColumn:
+            columns:
+              - column:
+                  name: generated_reports_information
+            tableName: download_status
--- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
+++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
@ -85,3 +85,7 @@ databaseChangeLog:
      file: db/changelog/34-add-reports-information-column.changelog.yaml
  - include:
      file: db/changelog/sql/35-update-skipped-color-existing-types.sql
+  - include:
+      file: db/changelog/36-revert-reports-information-column.changelog.yaml
+
+
--- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
+++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java
@ -1,5 +1,15 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;

+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
@ -10,7 +20,6 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
 import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
 import com.iqser.red.service.peristence.v1.server.service.download.DownloadReportMessageReceiver;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
@ -19,21 +28,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
 import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
 import com.iqser.red.storage.commons.service.StorageService;

-import lombok.Builder;
 import lombok.SneakyThrows;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.io.ByteArrayInputStream;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.assertj.core.api.Assertions.assertThat;

 public class DownloadPreparationTest extends AbstractPersistenceServerServiceTest {

@ -132,7 +127,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
        ReportResultMessage reportResultMessage = new ReportResultMessage();
        reportResultMessage.setUserId("1");
        reportResultMessage.setDownloadId(statuses.iterator().next().getStorageId());
-        reportResultMessage.setStoredFileInformation(sivList);

        downloadReportMessageReceiver.receive(objectMapper.writeValueAsString(reportResultMessage));
    }
--- a/persistence-service-v1/pom.xml
+++ b/persistence-service-v1/pom.xml
@ -28,7 +28,7 @@
        <redaction-service.version>3.114.0</redaction-service.version>
        <search-service.version>2.36.0</search-service.version>
        <pdftron-redaction-service.version>3.112.0</pdftron-redaction-service.version>
-        <redaction-report-service.version>3.47.0</redaction-report-service.version>
+        <redaction-report-service.version>3.52.0</redaction-report-service.version>
    </properties>

    <dependencyManagement>