From edfba898d98747eb74413738aa91e7461647b43d Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Mon, 2 May 2022 22:52:03 +0300 Subject: [PATCH] RED-3899 - dossier stats by template fixes and pseudo-test --- .../dossiertemplate/DossierTemplateStats.java | 35 +++++- .../repository/DossierRepository.java | 6 +- .../repository/FileRepository.java | 10 +- .../service/DossierTemplateStatsService.java | 14 +-- .../DossierTemplateTesterAndProvider.java | 10 +- .../service/DossierTesterAndProvider.java | 21 ++-- .../service/FileTesterAndProvider.java | 18 ++- .../tests/DossierTemplateStatsTest.java | 110 +++++++++++++++--- 8 files changed, 173 insertions(+), 51 deletions(-) diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java index e13d24566..03f5bd898 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java @@ -8,9 +8,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; @Data @Builder @@ -32,9 +30,36 @@ public class DossierTemplateStats { private int numberOfPages; private int numberOfExcludedPages; - private Map fileCountPerProcessingStatus = new HashMap<>(); - private Map fileCountPerWorkflowStatus = new HashMap<>(); - private Map dossierCountByStatus = new HashMap<>(); + private List fileCountPerProcessingStatus = new ArrayList<>(); + private List fileCountPerWorkflowStatus = new ArrayList<>(); + private List dossierCountByStatus = new ArrayList<>(); private List dossiersInTemplate = new ArrayList<>(); + + @Data + @AllArgsConstructor + public static class DossierStatusCount { + + private String statusId; + private int count; + + } + + @Data + @AllArgsConstructor + public static class ProcessingStatusCount { + + private ProcessingStatus processingStatus; + private int count; + + } + + @Data + @AllArgsConstructor + public static class WorkflowStatusCount { + + private WorkflowStatus workflowStatus; + private int count; + + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 1c7b746c9..25ffc7f5e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -50,9 +50,9 @@ public interface DossierRepository extends JpaRepository @Query("select count(d) from DossierEntity d where d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null and d.dossierTemplateId = :dossierTemplateId") int countActive(String dossierTemplateId); - @Query("select d.dossierStatusId, count(d) from DossierEntity d where d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null and d.dossierTemplateId = :dossierTemplateId group by d.dossierStatusId") + @Query("select d.dossierStatusId as dossierStatusId, count(d) as count from DossierEntity d where d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null and d.dossierTemplateId = :dossierTemplateId group by d.dossierStatusId") List countByDossierStatus(String dossierTemplateId); - @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId") - List findDossierIdsForTemplate(String dossierTemplateId); + @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null") + List findActiveDossierIdsForTemplate(String dossierTemplateId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 389f2743c..716140c10 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -159,7 +159,7 @@ public interface FileRepository extends JpaRepository { @Query("select f from FileEntity f where f.deleted is not null and f.hardDeletedTime is null and f.dossierId in :dossierIds") List getSoftDeletedFiles(List dossierIds); - @Query("select f.processingStatus, count(f) from FileEntity f " + + @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + @@ -167,16 +167,16 @@ public interface FileRepository extends JpaRepository { "group by f.processingStatus ") List countFilesByProcessingStatus(String dossierTemplateId); - @Query("select f.workflowStatus, count(f) from FileEntity f " + + @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + "inner join DossierEntity d on d.id = f.dossierId " + "where f.deleted is null and f.hardDeletedTime is null " + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + "and d.dossierTemplateId = :dossierTemplateId " + - "group by f.processingStatus ") + "group by f.workflowStatus ") List countFilesByWorkflowStatus(String dossierTemplateId); - @Query(value = "select sum(number_of_pages) sum_of_analyzed_pages, sum(json_array_length(cast(excluded_pages AS json))) as sum_of_excluded_pages " + - " from file join dossier on file.dossierId = dossier.id where file.deleted is null and file.hard_deleted_time is null " + + @Query(value = "select sum(number_of_pages) as numberOfAnalyzedPages, sum(json_array_length(cast(excluded_pages AS json))) as numberOfExcludedPages " + + " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) FilePageCountsProjection countPages(String dossierTemplateId); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java index 3900a786a..69a6f9a66 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java @@ -2,9 +2,6 @@ package com.iqser.red.service.peristence.v1.server.service; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierCountByStatusProjection; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; @@ -97,11 +94,14 @@ public class DossierTemplateStatsService { dts.setNumberOfPages(pageCounts.getNumberOfAnalyzedPages()); dts.setNumberOfExcludedPages(pageCounts.getNumberOfExcludedPages()); - dts.setFileCountPerProcessingStatus(processingCounts.stream().collect(Collectors.toMap(FileProcessingStatusProjection::getProcessingStatus, FileProcessingStatusProjection::getCount))); - dts.setFileCountPerWorkflowStatus(workflowCounts.stream().collect(Collectors.toMap(FileWorkflowStatusProjection::getWorkflowStatus, FileWorkflowStatusProjection::getCount))); - dts.setDossierCountByStatus(dossierStatusCounts.stream().collect(Collectors.toMap(DossierCountByStatusProjection::getDossierStatusId, DossierCountByStatusProjection::getCount))); + dts.setFileCountPerProcessingStatus(processingCounts.stream() + .map(t -> new DossierTemplateStats.ProcessingStatusCount(t.getProcessingStatus(), t.getCount())).collect(Collectors.toList())); + dts.setFileCountPerWorkflowStatus(workflowCounts.stream() + .map(t -> new DossierTemplateStats.WorkflowStatusCount(t.getWorkflowStatus(), t.getCount())).collect(Collectors.toList())); + dts.setDossierCountByStatus(dossierStatusCounts.stream() + .map(t -> new DossierTemplateStats.DossierStatusCount(t.getDossierStatusId(), t.getCount())).collect(Collectors.toList())); - dts.setDossiersInTemplate(dossierRepository.findDossierIdsForTemplate(dossierTemplateId)); + dts.setDossiersInTemplate(dossierRepository.findActiveDossierIdsForTemplate(dossierTemplateId)); return dts; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java index 579ee2ed2..c1ab54075 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java @@ -31,10 +31,10 @@ public class DossierTemplateTesterAndProvider { @Autowired private LegalBasisClient legalBasisClient; - public DossierTemplate provideTestTemplate() { + public DossierTemplate provideTestTemplate(String name) { CreateOrUpdateDossierTemplateRequest cru = new CreateOrUpdateDossierTemplateRequest(); cru.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL)); - cru.setName("Template 1"); + cru.setName(name); cru.setDescription("Template 1"); cru.setRequestingUser("1"); cru.setValidFrom(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); @@ -42,7 +42,7 @@ public class DossierTemplateTesterAndProvider { DossierTemplate result = dossierTemplateClient.createOrUpdateDossierTemplate(cru); - assertThat(result.getName()).isEqualTo("Template 1"); + assertThat(result.getName()).isEqualTo(name); DossierTemplate loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getId()); @@ -56,5 +56,9 @@ public class DossierTemplateTesterAndProvider { return loadedTemplate; } + public DossierTemplate provideTestTemplate() { + return provideTestTemplate("Template 1"); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java index a24ff0a96..26eda785b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java @@ -34,6 +34,19 @@ public class DossierTesterAndProvider { } public Dossier provideTestDossier(DossierTemplate testTemplate, String dossierName, DossierStatusInfo dossierStatus) { + + var result = provideTestDossierQuick(testTemplate,dossierName,dossierStatus); + + assertThat(result.getDossierName()).isEqualTo(dossierName); + + Dossier loadedDossier = dossierClient.getDossierById(result.getId(), false, false); + + assertThat(loadedDossier).isEqualTo(result); + + return loadedDossier; + } + + public Dossier provideTestDossierQuick(DossierTemplate testTemplate, String dossierName, DossierStatusInfo dossierStatus) { CreateOrUpdateDossierRequest cru = new CreateOrUpdateDossierRequest(); cru.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL)); cru.setDossierName(dossierName); @@ -52,13 +65,7 @@ public class DossierTesterAndProvider { Dossier result = dossierClient.addDossier(cru); - assertThat(result.getDossierName()).isEqualTo(dossierName); - - Dossier loadedDossier = dossierClient.getDossierById(result.getId(), false, false); - - assertThat(loadedDossier).isEqualTo(result); - - return loadedDossier; + return result; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index 6967ad213..aaab8505c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -48,11 +48,8 @@ public class FileTesterAndProvider { @SneakyThrows public FileModel testAndProvideFile(Dossier dossier, String fileName) { - - BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), fileName, dossier.getId(), "1"); - JSONPrimitive uploadResult = uploadClient.upload(upload); - - var file = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue()); + var fileId = testAndProvideFileQuick(dossier, fileName); + var file = fileClient.getFileStatus(dossier.getId(), fileId); assertThat(file.getId()).isNotBlank(); @@ -66,4 +63,15 @@ public class FileTesterAndProvider { return file; } + @SneakyThrows + public String testAndProvideFileQuick(Dossier dossier, String fileName) { + + + BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), fileName, dossier.getId(), "1"); + JSONPrimitive uploadResult = uploadClient.upload(upload); + + return uploadResult.getValue(); + + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java index 8e1bc1da1..58d11215d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java @@ -1,6 +1,23 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; +import com.google.common.collect.Sets; +import com.iqser.red.service.peristence.v1.server.integration.client.*; +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateDictionaryStats; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -10,20 +27,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.*; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import com.google.common.collect.Sets; -import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateStatsClient; -import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; -import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import static org.assertj.core.api.Assertions.assertThat; public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTest { private static final String TYPE_ID_1 = "type1"; @@ -33,6 +37,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe @Autowired private DossierTesterAndProvider dossierTesterAndProvider; + @Autowired + private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider; + @Autowired private DictionaryClient dictionaryClient; @@ -42,8 +49,25 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe @Autowired private DossierTemplateStatsClient dossierTemplateStatsClient; + @Autowired + private DossierClient dossierClient; + + @Autowired + private FileClient fileClient; + + @Autowired + private UploadClient uploadClient; + + @Autowired + private FileTesterAndProvider fileTesterAndProvider; + @Autowired + private FileStatusPersistenceService fileStatusPersistenceService; + + @Autowired + private DossierStatusClient dossierStatusClient; + @Test - public void testDossierTemplateStats() { + public void testDossierTemplateDictionaryStats() { var dossier = dossierTesterAndProvider.provideTestDossier(); @@ -211,4 +235,58 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe return dossierTemplateClient.createOrUpdateDossierTemplate(cru); } + + @Test + public void testDossierTemplateStats() { + + for (int i = 0; i < 2; i++) { + var template = dossierTemplateTesterAndProvider.provideTestTemplate("test template: " + i); + + var status = dossierStatusClient.createOrUpdateDossierStatus(CreateOrUpdateDossierStatusRequest.builder() + .dossierTemplateId(template.getId()).name("test").rank(100).build()); + for (int j = 0; j < 8; j++) { + + var dossier = dossierTesterAndProvider.provideTestDossierQuick(template, "test dossier: " + j + " - " + i, j % 2 == 0 ? status : null); + + for (int k = 0; k < 12; k++) { + var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k); + + if (k % 2 == 0) { + fileStatusPersistenceService.updateProcessingStatus(fileId, k, + 0L, 0L, 0L, 0L, 0L, 1, 1); + fileClient.excludePages(dossier.getId(), fileId, Set.of(k)); + } + if (k % 3 == 0) { + uploadClient.deleteFile(dossier.getId(), fileId); + } + if (j % 5 == 0) { + uploadClient.hardDeleteFiles(dossier.getId(), Set.of(fileId)); + } + if (j % 7 == 0) { + fileClient.setStatusUnderReview(dossier.getId(), fileId, "x"); + } + if (j % 11 == 0) { + fileClient.setStatusUnderApproval(dossier.getId(), fileId, "x"); + } + } + + if (j % 3 == 0) { + dossierClient.archiveDossiers(Set.of(dossier.getId())); + } + if (j % 5 == 0) { + dossierClient.delete(dossier.getId()); + } + if (j % 7 == 0) { + dossierClient.hardDeleteDossiers(Set.of(dossier.getId())); + } + + } + } + + long t1 = System.currentTimeMillis(); + var stats = dossierTemplateStatsClient.getDossierTemplateStats(); + assertThat(stats.size()).isEqualTo(2); + long t2 = System.currentTimeMillis(); + System.out.println("Time for Stats: " + (t2 - t1) + "ms"); + } }