RED-9496 - Implement graceful shutdown

RED-9496 - Implement graceful shutdown

See merge request redactmanager/redaction-report-service!83

redaction-report-service-v1/redaction-report-service-api-v1/build.gradle.kts

@@ -5,7 +5,7 @@ plugins {
 
 description = "redaction-report-service-api-v1"
 
-val persistenceServiceVersion = "2.380.0"
+val persistenceServiceVersion = "2.465.1"
 
 dependencies {
     implementation("io.github.openfeign:feign-core:12.2")

redaction-report-service-v1/redaction-report-service-api-v1/src/main/java/com/iqser/red/service/redaction/report/v1/api/model/ReportRequestMessage.java

@@ -23,6 +23,8 @@ public class ReportRequestMessage {
     private String dossierId;
 
     private String dossierTemplateId;
+
+    private String tenantId;
     
     @Builder.Default
     private List<String> fileIds = new ArrayList<>();

redaction-report-service-v1/redaction-report-service-server-v1/build.gradle.kts

@@ -16,7 +16,8 @@ val springCommonsVersion = "2.1.0"
 val storageCommonsVersion = "2.45.0"
 val poiVersion = "5.2.3"
 val metricCommonsVersion = "2.1.0"
-val persistenceServiceVersion = "2.420.0"
+val lifecycleCommonsVersion = "0.6.0"
+val persistenceServiceVersion = "2.465.1"
 val springBootStarterVersion = "3.2.3"
 
 configurations {
@@ -35,6 +36,7 @@ dependencies {
     implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}")
 
     implementation("com.knecon.fforesight:tenant-commons:${tenantCommonVersion}")
+    implementation("com.knecon.fforesight:lifecycle-commons:${lifecycleCommonsVersion}")
     implementation("com.iqser.red.commons:storage-commons:${storageCommonsVersion}")
     implementation("com.iqser.red.commons:spring-commons:${springCommonsVersion}")
     implementation("com.iqser.red.commons:metric-commons:${metricCommonsVersion}")

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/Application.java

@@ -12,6 +12,7 @@ import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfi
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 import org.springframework.scheduling.annotation.EnableAsync;
@@ -21,6 +22,7 @@ import com.iqser.red.service.redaction.report.v1.server.client.DossierClient;
 import com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration;
 import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
@@ -30,10 +32,11 @@ import io.micrometer.core.instrument.MeterRegistry;
 @EnableAsync
 @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class})
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
-@Import({MessagingConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
+@Import({MessagingConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class, LifecycleAutoconfiguration.class})
 @EnableFeignClients(basePackageClasses = {DossierClient.class})
 @EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
 @EnableConfigurationProperties(ReportTemplateSettings.class)
+@EnableAspectJAutoProxy
 public class Application {
 
     /**

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/EntityLogConverterService.java

@@ -103,6 +103,10 @@ public class EntityLogConverterService {
                         return;
                     }
 
+                    if (entry.getState() == EntryState.PENDING) {
+                        return;
+                    }
+
                     if (entry.isExcluded()) {
                         return;
                     }

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ExcelReportTemplateService.java

@@ -44,6 +44,7 @@ public class ExcelReportTemplateService {
                                                                                        PlaceholderModel placeholderModel,
                                                                                        String templateName,
                                                                                        String downloadId,
+                                                                                       String tenantId,
                                                                                        List<ReportRedactionEntry> reportEntries,
                                                                                        ReportTemplate reportTemplate) {
 
@@ -68,7 +69,7 @@ public class ExcelReportTemplateService {
                                                                      excelModel,
                                                                      true);
             byte[] template = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
-            return reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, template)
+            return reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, tenantId, template)
                     .thenApply(storageId -> new StoredFileInformation(fileStatus.getId(), storageId, ReportType.EXCEL_TEMPLATE_SINGLE_FILE, reportTemplate.getTemplateId(), 0));
         } catch (IOException e) {
             CompletableFuture<StoredFileInformation> finalResultFuture = new CompletableFuture<>();

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportGenerationService.java

@@ -7,9 +7,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
-import org.apache.poi.ss.usermodel.Sheet;
-import org.apache.poi.xssf.streaming.SXSSFWorkbook;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.apache.poi.xwpf.usermodel.XWPFDocument;
 import org.springframework.stereotype.Service;
 
@@ -29,7 +26,6 @@ import com.iqser.red.service.redaction.report.v1.server.model.ReportTemplatesMod
 import com.iqser.red.service.redaction.report.v1.server.settings.ReportTemplateSettings;
 import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageService;
 import com.iqser.red.service.redaction.report.v1.server.storage.ReportStorageServiceAsyncWrapper;
-import com.iqser.red.service.redaction.report.v1.server.utils.TemplateCache;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -66,6 +62,7 @@ public class ReportGenerationService {
         ReportTemplatesModel reportTemplatesModel = reportTemplateService.prepareReportTemplates(reportMessage);
         var placeholderModel = generatePlaceholderService.buildPlaceholders(dossier);
         String downloadId = reportMessage.getDownloadId();
+        String tenantId = reportMessage.getTenantId();
 
         List<CompletableFuture<Void>> allFutures = new ArrayList<>();
 
@@ -92,6 +89,7 @@ public class ReportGenerationService {
                                                                                                    placeholderModel,
                                                                                                    isLastFile,
                                                                                                    downloadId,
+                                                                                                   tenantId,
                                                                                                    fileStatus,
                                                                                                    reportEntries);
             allFutures.add(multiFileWordReportsFuture);
@@ -101,6 +99,7 @@ public class ReportGenerationService {
                                                                                             dossier,
                                                                                             placeholderModel,
                                                                                             downloadId,
+                                                                                            tenantId,
                                                                                             fileStatus,
                                                                                             reportEntries);
             allFutures.add(singleFileReportsAsync);
@@ -111,7 +110,7 @@ public class ReportGenerationService {
 
         for (MultiFileWorkbook multiFileWorkbook : reportTemplatesModel.multiFileWorkbookReportTemplates) {
             byte[] template = excelTemplateReportGenerationService.toByteArray(multiFileWorkbook.getWriteWorkbook());
-            CompletableFuture<Void> future = reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, template).thenAccept(storageId -> {
+            CompletableFuture<Void> future = reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, tenantId, template).thenAccept(storageId -> {
                 storedFileInformation.add(new StoredFileInformation(null, storageId, ReportType.EXCEL_TEMPLATE_MULTI_FILE, multiFileWorkbook.getTemplateId(), 0));
             });
             allFutures.add(future);
@@ -119,7 +118,7 @@ public class ReportGenerationService {
 
         for (MultiFileDocument multiFileDocument : reportTemplatesModel.multiFileDocumentReportTemplates) {
             byte[] template = wordReportGenerationService.toByteArray(multiFileDocument.getDocument());
-            CompletableFuture<Void> future = reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, template).thenAccept(storageId -> {
+            CompletableFuture<Void> future = reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, tenantId, template).thenAccept(storageId -> {
                 storedFileInformation.add(new StoredFileInformation(null,
                                                                     storageId,
                                                                     ReportType.WORD_TEMPLATE_MULTI_FILE,
@@ -131,7 +130,7 @@ public class ReportGenerationService {
 
         CompletableFuture.allOf(allFutures.toArray(new CompletableFuture[0])).join();
 
-        return reportStorageService.storeReportInformation(reportMessage.getDownloadId(), storedFileInformation);
+        return reportStorageService.storeReportInformation(downloadId, tenantId, storedFileInformation);
     }
 
 
@@ -164,6 +163,7 @@ public class ReportGenerationService {
                                                                       PlaceholderModel placeholderModel,
                                                                       boolean isLastFile,
                                                                       String downloadId,
+                                                                      String tenantId,
                                                                       FileModel fileStatus,
                                                                       List<ReportRedactionEntry> reportEntries) {
 
@@ -177,7 +177,7 @@ public class ReportGenerationService {
                 wordReportGenerationService.removePlaceholdersRow(wordReportGenerationService.getRedactionTable(multiFileDocument.getDocument()));
                 byte[] wordDoc = wordReportGenerationService.toByteArray(multiFileDocument.getDocument());
 
-                CompletableFuture<Void> future = reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, wordDoc).thenAccept(storageId -> {
+                CompletableFuture<Void> future = reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, tenantId, wordDoc).thenAccept(storageId -> {
                     storedFileInformation.add(new StoredFileInformation(null,
                                                                         storageId,
                                                                         ReportType.WORD_TEMPLATE_MULTI_FILE,
@@ -217,17 +217,19 @@ public class ReportGenerationService {
                                                                    Dossier dossier,
                                                                    PlaceholderModel placeholderModel,
                                                                    String downloadId,
+                                                                   String tenantId,
                                                                    FileModel fileStatus,
                                                                    List<ReportRedactionEntry> reportEntries) {
 
         return CompletableFuture.allOf(singleFilesTemplates.stream()
                                                .map(reportTemplate -> reportTemplateService.createReportFromTemplateAsync(dossier,
-                                                                                                    fileStatus,
+                                                                                                                          fileStatus,
-                                                                                                    placeholderModel,
+                                                                                                                          placeholderModel,
-                                                                                                    reportTemplate.getFileName(),
+                                                                                                                          reportTemplate.getFileName(),
-                                                                                                    downloadId,
+                                                                                                                          downloadId,
-                                                                                                    reportEntries,
+                                                                                                                          tenantId,
-                                                                                                    reportTemplate).thenAccept(storedFileInformation::add))
+                                                                                                                          reportEntries,
+                                                                                                                          reportTemplate).thenAccept(storedFileInformation::add))
                                                .toArray(CompletableFuture[]::new));
     }
 

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportMessageReceiver.java

@@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.report.v1.server.service;
 import static com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration.REPORT_QUEUE;
 import static com.iqser.red.service.redaction.report.v1.server.configuration.MessagingConfiguration.REPORT_RESULT_QUEUE;
 
+import java.util.concurrent.locks.ReentrantLock;
+
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
@@ -27,6 +29,7 @@ public class ReportMessageReceiver {
     private final ReportGenerationService reportGenerationService;
     private final RabbitTemplate rabbitTemplate;
 
+    private final ReentrantLock lock = new ReentrantLock();
 
     @SneakyThrows
     @RabbitHandler
@@ -43,8 +46,13 @@ public class ReportMessageReceiver {
         long start = System.currentTimeMillis();
         log.info("Start generating reports for downloadId {}", reportMessage.getDownloadId());
 
-        var reportFileInformationStorageId = reportGenerationService.generateReports(reportMessage);
+        lock.lock();
-        addToReportResultQueue(reportMessage.getUserId(), reportMessage.getDownloadId(), reportFileInformationStorageId);
+        try {
+            String reportFileInformationStorageId = reportGenerationService.generateReports(reportMessage);
+            addToReportResultQueue(reportMessage.getUserId(), reportMessage.getDownloadId(), reportFileInformationStorageId);
+        } finally {
+            lock.unlock();
+        }
 
         long end = System.currentTimeMillis();
         log.info("Successfully generated reports for downloadId {}, took {}", reportMessage.getDownloadId(), end - start);

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/ReportTemplateService.java

@@ -63,13 +63,14 @@ public class ReportTemplateService {
                                                                                    PlaceholderModel placeholderModel,
                                                                                    String templateName,
                                                                                    String downloadId,
+                                                                                   String tenantId,
                                                                                    List<ReportRedactionEntry> reportEntries,
                                                                                    ReportTemplate reportTemplate) {
 
         if (reportTemplate.getFileName().endsWith(XLSX_EXTENSION)) {
-            return excelReportTemplateService.createExcelReportFromTemplateAsync(dossier, fileStatus, placeholderModel, templateName, downloadId, reportEntries, reportTemplate);
+            return excelReportTemplateService.createExcelReportFromTemplateAsync(dossier, fileStatus, placeholderModel, templateName, downloadId, tenantId, reportEntries, reportTemplate);
         } else {
-            return wordReportTemplateService.createWordReportFromTemplateAsync(dossier, fileStatus, placeholderModel, templateName, downloadId, reportEntries, reportTemplate);
+            return wordReportTemplateService.createWordReportFromTemplateAsync(dossier, fileStatus, placeholderModel, templateName, downloadId, tenantId, reportEntries, reportTemplate);
         }
     }
 

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/service/WordReportTemplateService.java

@@ -40,6 +40,7 @@ public class WordReportTemplateService {
                                                                                        PlaceholderModel placeholderModel,
                                                                                        String templateName,
                                                                                        String downloadId,
+                                                                                       String tenantId,
                                                                                        List<ReportRedactionEntry> reportEntries,
                                                                                        ReportTemplate reportTemplate) {
 
@@ -48,7 +49,7 @@ public class WordReportTemplateService {
             XWPFDocument doc = new XWPFDocument(is);
             wordReportGenerationService.generateWordReport(reportEntries, placeholderModel, templateName, doc, fileStatus, dossier, true);
             byte[] template = wordReportGenerationService.toByteArray(doc);
-            return reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, template)
+            return reportStorageServiceAsyncWrapper.storeObjectAsync(downloadId, tenantId, template)
                     .thenApply(storageId -> new StoredFileInformation(fileStatus.getId(), storageId, ReportType.WORD_SINGLE_FILE, reportTemplate.getTemplateId(), 0));
         } catch (IOException e) {
             CompletableFuture<StoredFileInformation> finalResultFuture = new CompletableFuture<>();

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/ReportStorageService.java

@@ -23,9 +23,11 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class ReportStorageService {
 
     private final StorageService storageService;
@@ -33,19 +35,21 @@ public class ReportStorageService {
     private final EntityLogMongoService entityLogMongoService;
 
 
-    public String storeObject(String downloadId, byte[] data) {
+    public String storeObject(String downloadId, String tenantId, byte[] data) {
 
         String storageId = StorageIdUtils.getStorageId(downloadId, UUID.randomUUID().toString());
-        storageService.storeObject(TenantContext.getTenantId(), storageId, new ByteArrayInputStream(data));
+        log.info("Async storing object storageId {} tenantId {}", storageId, tenantId);
+        storageService.storeObject(tenantId, storageId, new ByteArrayInputStream(data));
         return storageId;
     }
 
 
     @SneakyThrows
-    public String storeReportInformation(String downloadId, List<StoredFileInformation> storedFileInformations) {
+    public String storeReportInformation(String downloadId, String tenantId, List<StoredFileInformation> storedFileInformations) {
 
         String storageId = StorageIdUtils.getStorageId(downloadId, "REPORT_INFO.json");
-        storageService.storeJSONObject(TenantContext.getTenantId(), storageId, storedFileInformations);
+        log.info("Storing report information with storageId {} tenantId {}", storageId, tenantId);
+        storageService.storeJSONObject(tenantId, storageId, storedFileInformations);
         return storageId;
     }
 
@@ -76,13 +80,16 @@ public class ReportStorageService {
         return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType));
     }
 
+
     @SneakyThrows
     public InputStream getObject(String tenantId, String storageId) {
+
         File destFile = File.createTempFile("destFile", ".data");
         storageService.downloadTo(tenantId, storageId, destFile);
         return Files.newInputStream(Paths.get(destFile.getPath()), StandardOpenOption.DELETE_ON_CLOSE);
     }
 
+
     public EntityLog getEntityLog(String dossierId, String fileId, List<String> excludedTypes) {
 
         Optional<EntityLog> entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(dossierId, fileId);

redaction-report-service-v1/redaction-report-service-server-v1/src/main/java/com/iqser/red/service/redaction/report/v1/server/storage/ReportStorageServiceAsyncWrapper.java

@@ -19,7 +19,7 @@ public class ReportStorageServiceAsyncWrapper {
     private final Executor ioExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
 
-    public CompletableFuture<String> storeObjectAsync(String downloadId, byte[] data) {
+    public CompletableFuture<String> storeObjectAsync(String downloadId, String tenantId, byte[] data) {
-        return CompletableFuture.supplyAsync(() -> reportStorageService.storeObject(downloadId, data), ioExecutor);
+        return CompletableFuture.supplyAsync(() -> reportStorageService.storeObject(downloadId, tenantId, data), ioExecutor);
     }
 }

redaction-report-service-v1/redaction-report-service-server-v1/src/main/resources/application.yml

@@ -8,6 +8,9 @@ fforesight.tenants.remote: true
 server:
   port: 8080
 
+lifecycle:
+  base-package: com.iqser.red.service.redaction.report.v1.server
+
 logging.pattern.level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"
 
 logging.type: ${LOGGING_TYPE:CONSOLE}

redaction-report-service-v1/redaction-report-service-server-v1/src/test/java/com/iqser/red/service/redaction/report/v1/server/RedactionReportIntegrationTest.java

@@ -527,6 +527,39 @@ public class RedactionReportIntegrationTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    public void testExcelReportWithPendingEntries() {
+
+        Dossier dossier = prepareDossier();
+        EntityLog entityLog = objectMapper.readValue(new ClassPathResource("files/entityLogWithPendingEntries.json").getInputStream(), EntityLog.class);
+        List<EntityLogLegalBasis> legalBasisMapping = objectMapper.readValue(new ClassPathResource("files/legalBasisMapping.json").getInputStream(), new TypeReference<>() {
+        });
+        Map<String, String> mapOfEntityDisplayName = createEntityDisplayNames(entityLog);
+        List<ReportRedactionEntry> reportEntries = entityLogConverterService.convertAndSort(DOSSIER_ID, FILE_ID,entityLog, legalBasisMapping, mapOfEntityDisplayName);
+        var imageResource = new ClassPathResource("files/exampleImage.jpg");
+
+        FileModel fileModel = FileModel.builder().filename("filename").build();
+
+        ClassPathResource templateResource = new ClassPathResource("templates/Excel Report_PlaceholderTest.xlsx");
+
+        var placeholders = buildPlaceHolderModel(Map.of("{{dossier_attribute.test1}}", "replaced_dossier_test1"),
+                                                 Map.of("{{file_attribute.test1}}", "replaced_file_test1", "{{file_attribute.test2}}", "replaced_file_test2"),
+                                                 List.of(new ImagePlaceholder("{{dossier.attribute.Signature}}", IOUtils.toByteArray(imageResource.getInputStream()))));
+
+        XSSFWorkbook readWorkbook = new XSSFWorkbook(templateResource.getInputStream());
+        var excelModel = excelTemplateReportGenerationService.calculateExcelModel(readWorkbook.getSheetAt(0), dossier.getDossierTemplateId());
+        SXSSFWorkbook writeWorkbook = new SXSSFWorkbook();
+        writeWorkbook.createSheet("Sheet1");
+        excelTemplateReportGenerationService.generateExcelReport(reportEntries, placeholders, "test", writeWorkbook, "dossierName", fileModel, excelModel, true);
+
+        byte[] excelTemplateReport3 = excelTemplateReportGenerationService.toByteArray(writeWorkbook);
+        try (FileOutputStream fileOutputStream = new FileOutputStream( getTemporaryDirectory() + "/Report_Without_Pending_Entries.xlsx")) {
+            fileOutputStream.write(excelTemplateReport3);
+        }
+    }
+
+
     @Test
     @SneakyThrows
     public void testScmReport() {