Pull request #475: RED-4527: Store information of generated reports in storage

Merge in RED/persistence-service from RED-4527 to master

* commit 'a775d52cad4890b3d1be3ff942617a8dc8d6bf42':
  RED-4527: Store information of generated reports in storage

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java

@@ -23,10 +23,8 @@ import org.hibernate.annotations.FetchMode;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
-import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStoredFileInformationConverter;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
 import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
-import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -74,8 +72,4 @@ public class DownloadStatusEntity {
     @Convert(converter = JSONDownloadFileTypeConverter.class)
     Set<DownloadFileType> downloadFileTypes = new HashSet<>();
 
-    @Builder.Default
-    @Column(columnDefinition = "text", name = "generated_reports_information")
-    @Convert(converter = JSONStoredFileInformationConverter.class)
-    Set<StoredFileInformation> generatedReportsInformation = new HashSet<>();
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.peristence.v1.server.service.download;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -15,6 +14,7 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionMessage;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionResultDetail;
@@ -41,6 +41,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -66,8 +67,6 @@ public class DownloadPreparationService {
     public void createDownload(ReportResultMessage reportResultMessage) throws JsonProcessingException {
 
         DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
-        downloadStatus.setGeneratedReportsInformation(new HashSet<>(reportResultMessage.getStoredFileInformation()));
-        downloadStatusPersistenceService.updateStatusEntity(downloadStatus);
 
         DossierEntity dossier = downloadStatus.getDossier();
 
@@ -112,10 +111,12 @@ public class DownloadPreparationService {
 
         DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(reportResultMessage.getDownloadId());
 
+        var storedFileInformations = getStoredFileInformation(reportResultMessage.getDownloadId());
+
         try (FileSystemBackedArchiver fileSystemBackedArchiver = new FileSystemBackedArchiver()) {
 
             generateAndAddFiles(downloadStatus, reportResultMessage, fileSystemBackedArchiver);
-            addReports(downloadStatus, fileSystemBackedArchiver);
+            addReports(reportResultMessage.getDownloadId(), storedFileInformations, fileSystemBackedArchiver);
             storeZipFile(downloadStatus, fileSystemBackedArchiver);
 
             downloadStatusPersistenceService.updateStatus(downloadStatus.getStorageId(), DownloadStatusValue.READY, fileSystemBackedArchiver.getContentLength());
@@ -130,7 +131,7 @@ public class DownloadPreparationService {
                     .build());
         }
 
-        downloadReportCleanupService.deleteTmpReportFiles(downloadStatus.getGeneratedReportsInformation()
+        downloadReportCleanupService.deleteTmpReportFiles(storedFileInformations
                 .stream()
                 .map(StoredFileInformation::getStorageId)
                 .collect(Collectors.toSet()));
@@ -210,11 +211,11 @@ public class DownloadPreparationService {
     }
 
 
-    private void addReports(DownloadStatusEntity downloadStatus, FileSystemBackedArchiver fileSystemBackedArchiver) {
+    private void addReports(String downloadId, List<StoredFileInformation> storedFileInformations, FileSystemBackedArchiver fileSystemBackedArchiver) {
 
         long addReportsStart = System.currentTimeMillis();
 
-        for (StoredFileInformation storedFileInformation : downloadStatus.getGeneratedReportsInformation()) {
+        for (StoredFileInformation storedFileInformation : storedFileInformations) {
 
             FileEntity fileStatus = null;
 
@@ -227,10 +228,17 @@ public class DownloadPreparationService {
             fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(removeExtension(reportTemplate.getFileName()) + (reportTemplate.isMultiFileReport() ? " (multifile)" : ""), addSuffix(createFileName(fileStatus, reportTemplate), "justification"), report));
 
         }
-        log.info("Successfully added {} reports for downloadId {}, took {}", downloadStatus.getGeneratedReportsInformation()
+        log.info("Successfully added {} reports for downloadId {}, took {}", storedFileInformations.size(), downloadId, System.currentTimeMillis() - addReportsStart);
-                .size(), downloadStatus.getStorageId(), System.currentTimeMillis() - addReportsStart);
     }
 
+    @SneakyThrows
+    private List<StoredFileInformation> getStoredFileInformation(String downloadId){
+
+        var storageId = downloadId.substring(0, downloadId.length() - 3) + "/REPORT_INFO.json";
+
+        return objectMapper.readValue(fileManagementStorageService.getStoredObjectBytes(storageId), new TypeReference<>() {
+        });
+    }
 
     private String createFileName(FileEntity fileStatus, ReportTemplateEntity reportTemplate) {
 

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/36-revert-reports-information-column.changelog.yaml

@@ -0,0 +1,10 @@
+databaseChangeLog:
+  - changeSet:
+      id: remove-reports-information-column
+      author: dom
+      changes:
+        - removeColumn:
+            columns:
+              - column:
+                  name: generated_reports_information
+            tableName: download_status

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -85,3 +85,7 @@ databaseChangeLog:
       file: db/changelog/34-add-reports-information-column.changelog.yaml
   - include:
       file: db/changelog/sql/35-update-skipped-color-existing-types.sql
+  - include:
+      file: db/changelog/36-revert-reports-information-column.changelog.yaml
+
+

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DownloadPreparationTest.java

@@ -1,5 +1,15 @@
 package com.iqser.red.service.peristence.v1.server.integration.tests;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
 import com.iqser.red.service.peristence.v1.server.integration.client.DownloadClient;
@@ -10,7 +20,6 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
 import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
 import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
 import com.iqser.red.service.peristence.v1.server.service.download.DownloadReportMessageReceiver;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ReportTemplateUploadRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
@@ -19,21 +28,7 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
 import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
 import com.iqser.red.storage.commons.service.StorageService;
 
-import lombok.Builder;
 import lombok.SneakyThrows;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.io.ByteArrayInputStream;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.assertj.core.api.Assertions.assertThat;
 
 public class DownloadPreparationTest extends AbstractPersistenceServerServiceTest {
 
@@ -132,7 +127,6 @@ public class DownloadPreparationTest extends AbstractPersistenceServerServiceTes
         ReportResultMessage reportResultMessage = new ReportResultMessage();
         reportResultMessage.setUserId("1");
         reportResultMessage.setDownloadId(statuses.iterator().next().getStorageId());
-        reportResultMessage.setStoredFileInformation(sivList);
 
         downloadReportMessageReceiver.receive(objectMapper.writeValueAsString(reportResultMessage));
     }

persistence-service-v1/pom.xml

@@ -28,7 +28,7 @@
         <redaction-service.version>3.114.0</redaction-service.version>
         <search-service.version>2.36.0</search-service.version>
         <pdftron-redaction-service.version>3.112.0</pdftron-redaction-service.version>
-        <redaction-report-service.version>3.47.0</redaction-report-service.version>
+        <redaction-report-service.version>3.52.0</redaction-report-service.version>
     </properties>
 
     <dependencyManagement>