diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateDictionaryStats.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateDictionaryStats.java new file mode 100644 index 000000000..ed9186690 --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateDictionaryStats.java @@ -0,0 +1,22 @@ +package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate; + +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DossierTemplateDictionaryStats { + + private String dossierTemplateId; + private int numberOfDictionaries; // number of Types for the dossierTemplate + private List dictionarySummaryList = new ArrayList<>(); + +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java index 57ebcd6f5..03f5bd898 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/DossierTemplateStats.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -14,7 +15,51 @@ import java.util.List; @NoArgsConstructor @AllArgsConstructor public class DossierTemplateStats { + private String dossierTemplateId; - private int numberOfDictionaries; // number of Types for the dossierTemplate - private List dictionarySummaryList = new ArrayList<>(); + private String name; + + private int numberOfDeletedDossiers; + private int numberOfActiveDossiers; + private int numberOfArchivedDossiers; + + private int numberOfPeople; + private int numberOfActiveFiles; + private int numberOfSoftDeletedFiles; + + private int numberOfPages; + private int numberOfExcludedPages; + + private List fileCountPerProcessingStatus = new ArrayList<>(); + private List fileCountPerWorkflowStatus = new ArrayList<>(); + private List dossierCountByStatus = new ArrayList<>(); + + private List dossiersInTemplate = new ArrayList<>(); + + @Data + @AllArgsConstructor + public static class DossierStatusCount { + + private String statusId; + private int count; + + } + + @Data + @AllArgsConstructor + public static class ProcessingStatusCount { + + private ProcessingStatus processingStatus; + private int count; + + } + + @Data + @AllArgsConstructor + public static class WorkflowStatusCount { + + private WorkflowStatus workflowStatus; + private int count; + + } } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java index 5983ba8a2..eff7ccdc4 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/DossierTemplateStatsResource.java @@ -1,7 +1,9 @@ package com.iqser.red.service.persistence.service.v1.api.resources; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateDictionaryStats; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStats; import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -12,6 +14,18 @@ public interface DossierTemplateStatsResource { String REST_PATH = "/dossier-template-stats"; + String DICTIONARY_PATH = "/dictionary"; + + String DOSSIER_TEMPLATE_ID = "dossierTemplateId"; + String DOSSIER_TEMPLATE_ID_PATH_VARIABLE = "/{" + DOSSIER_TEMPLATE_ID + "}"; + + @PostMapping(value = REST_PATH+DICTIONARY_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) + List getDossierTemplateDictionaryStats(@RequestBody Set dossierTemplateIds); + + @PostMapping(value = REST_PATH+DOSSIER_TEMPLATE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE ) + DossierTemplateStats getDossierTemplateStats(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId); + @PostMapping(value = REST_PATH, produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE) - List getDossierTemplateStats(@RequestBody Set dossierTemplateIds); + List getDossierTemplateStats(); + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index 0df67bf72..752c00503 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -180,4 +180,5 @@ public class DossierPersistenceService { throw new BadRequestException("Cannot unarchive dossier!"); } } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/DossierCountByStatusProjection.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/DossierCountByStatusProjection.java new file mode 100644 index 000000000..47462ae4d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/DossierCountByStatusProjection.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection; + +public interface DossierCountByStatusProjection { + + int getCount(); + + String getDossierStatusId(); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FilePageCountsProjection.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FilePageCountsProjection.java new file mode 100644 index 000000000..98c98c09a --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FilePageCountsProjection.java @@ -0,0 +1,8 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection; + +public interface FilePageCountsProjection { + + int getNumberOfExcludedPages(); + + int getNumberOfAnalyzedPages(); +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FileProcessingStatusProjection.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FileProcessingStatusProjection.java new file mode 100644 index 000000000..444087390 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FileProcessingStatusProjection.java @@ -0,0 +1,11 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection; + +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; + +public interface FileProcessingStatusProjection { + + int getCount(); + + ProcessingStatus getProcessingStatus(); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FileWorkflowStatusProjection.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FileWorkflowStatusProjection.java new file mode 100644 index 000000000..96fa4d47f --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/projection/FileWorkflowStatusProjection.java @@ -0,0 +1,11 @@ +package com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection; + +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; + +public interface FileWorkflowStatusProjection { + + int getCount(); + + WorkflowStatus getWorkflowStatus(); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 9f0a2fe15..25ffc7f5e 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -1,13 +1,13 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; -import java.time.OffsetDateTime; -import java.util.List; - +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.DossierCountByStatusProjection; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import java.time.OffsetDateTime; +import java.util.List; public interface DossierRepository extends JpaRepository { @@ -40,4 +40,19 @@ public interface DossierRepository extends JpaRepository @Modifying @Query("update DossierEntity d set d.lastUpdated = :lastUpdated, d.archivedTime = null where d.id = :dossierId") int unarchiveDossier(String dossierId, OffsetDateTime lastUpdated); + + @Query("select count(d) from DossierEntity d where d.archivedTime is not null and d.dossierTemplateId = :dossierTemplateId") + int countArchived(String dossierTemplateId); + + @Query("select count(d) from DossierEntity d where d.hardDeletedTime is null and d.softDeletedTime is not null and d.dossierTemplateId = :dossierTemplateId") + int countSofDeleted(String dossierTemplateId); + + @Query("select count(d) from DossierEntity d where d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null and d.dossierTemplateId = :dossierTemplateId") + int countActive(String dossierTemplateId); + + @Query("select d.dossierStatusId as dossierStatusId, count(d) as count from DossierEntity d where d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null and d.dossierTemplateId = :dossierTemplateId group by d.dossierStatusId") + List countByDossierStatus(String dossierTemplateId); + + @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null") + List findActiveDossierIdsForTemplate(String dossierTemplateId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 5631f5ab5..716140c10 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -1,15 +1,17 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; -import java.time.OffsetDateTime; -import java.util.List; - +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import java.time.OffsetDateTime; +import java.util.List; public interface FileRepository extends JpaRepository { @@ -147,7 +149,7 @@ public interface FileRepository extends JpaRepository { @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'DELETED' or f.processingStatus = 'ERROR' )" + - "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " ) + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null ") List getAllRelevantStatusesForReanalysisScheduler(); @Modifying(clearAutomatically = true) @@ -157,6 +159,33 @@ public interface FileRepository extends JpaRepository { @Query("select f from FileEntity f where f.deleted is not null and f.hardDeletedTime is null and f.dossierId in :dossierIds") List getSoftDeletedFiles(List dossierIds); + @Query("select f.processingStatus as processingStatus, count(f) as count from FileEntity f " + + "inner join DossierEntity d on d.id = f.dossierId " + + "where f.deleted is null and f.hardDeletedTime is null " + + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + + "and d.dossierTemplateId = :dossierTemplateId " + + "group by f.processingStatus ") + List countFilesByProcessingStatus(String dossierTemplateId); + + @Query("select f.workflowStatus as workflowStatus, count(f) as count from FileEntity f " + + "inner join DossierEntity d on d.id = f.dossierId " + + "where f.deleted is null and f.hardDeletedTime is null " + + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null " + + "and d.dossierTemplateId = :dossierTemplateId " + + "group by f.workflowStatus ") + List countFilesByWorkflowStatus(String dossierTemplateId); + + @Query(value = "select sum(number_of_pages) as numberOfAnalyzedPages, sum(json_array_length(cast(excluded_pages AS json))) as numberOfExcludedPages " + + " from file join dossier on file.dossier_id = dossier.id where file.deleted is null and file.hard_deleted_time is null " + + " and dossier.archived_time is null and dossier.soft_deleted_time is null and dossier.hard_deleted_time is null" + + " and dossier.dossier_template_id = :dossierTemplateId", nativeQuery = true) + FilePageCountsProjection countPages(String dossierTemplateId); + + @Query("select count(f) from FileEntity f inner join DossierEntity d on d.id = f.dossierId where " + + "f.hardDeletedTime is not null and f.deleted is not null and " + + "d.dossierTemplateId = :dossierTemplateId and " + + "d.softDeletedTime is null and d.hardDeletedTime is null and d.archivedTime is null") + int countActiveFiles(String dossierTemplateId); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateStatsController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateStatsController.java index 0c159bb78..1955cd696 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateStatsController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierTemplateStatsController.java @@ -1,6 +1,7 @@ package com.iqser.red.service.peristence.v1.server.controller; import com.iqser.red.service.peristence.v1.server.service.DossierTemplateStatsService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateDictionaryStats; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStats; import com.iqser.red.service.persistence.service.v1.api.resources.DossierTemplateStatsResource; import lombok.RequiredArgsConstructor; @@ -17,7 +18,17 @@ public class DossierTemplateStatsController implements DossierTemplateStatsResou private final DossierTemplateStatsService dossierTemplateStatsService; @Override - public List getDossierTemplateStats(@RequestBody Set dossierTemplateIds) { + public List getDossierTemplateDictionaryStats(@RequestBody Set dossierTemplateIds) { return dossierTemplateStatsService.getDossierTemplateStats(dossierTemplateIds); } + + @Override + public DossierTemplateStats getDossierTemplateStats(String dossierTemplateId) { + return dossierTemplateStatsService.getDossierTemplateStats(dossierTemplateId); + } + + @Override + public List getDossierTemplateStats() { + return dossierTemplateStatsService.getDossierTemplateStats(); + } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java index dfe95a802..69a6f9a66 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierTemplateStatsService.java @@ -1,7 +1,11 @@ package com.iqser.red.service.peristence.v1.server.service; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateDictionaryStats; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStats; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse; @@ -9,10 +13,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Slf4j @@ -21,25 +22,49 @@ import java.util.stream.Collectors; public class DossierTemplateStatsService { private final DictionaryPersistenceService dictionaryPersistenceService; - private final DossierTemplatePersistenceService dossierTemplatePersistenceService; - public List getDossierTemplateStats(Set dossierTemplateIds) { + private final DossierTemplateRepository dossierTemplateRepository; + + private final FileRepository fileRepository; + + private final DossierRepository dossierRepository; + + + public DossierTemplateStats getDossierTemplateStats(String dossierTemplateId) { + + var dossierTemplate = dossierTemplateRepository.findByIdAndNotDeleted(dossierTemplateId).orElseThrow(() -> new NotFoundException("Dossier Template with id: " + dossierTemplateId + " not found.")); + + return getDossierTemplateStats(dossierTemplateId, dossierTemplate.getName()); + } + + public List getDossierTemplateStats() { + var allUnDeletedTemplates = dossierTemplateRepository.findAllWhereDeletedIsFalse(); + + final List dossierTemplateStatsList = Collections.synchronizedList(new ArrayList<>()); + allUnDeletedTemplates.parallelStream().forEach(dt -> dossierTemplateStatsList.add(getDossierTemplateStats(dt.getId(), dt.getName()))); + + dossierTemplateStatsList.sort(Comparator.comparing(DossierTemplateStats::getName)); + + return dossierTemplateStatsList; + } + + public List getDossierTemplateStats(Set dossierTemplateIds) { List dictionarySummaryList = dictionaryPersistenceService.getDictionarySummaryForDossierTemplateId(dossierTemplateIds); Map> dictionarySummaryMap = dictionarySummaryList.stream().collect( Collectors.groupingBy(DictionarySummaryResponse::getDossierTemplateId, - Collectors.mapping(this::getDossierTemplateStats, Collectors.toList()))); - List dossierTemplateStatsList = dictionarySummaryMap.entrySet().stream() + Collectors.mapping(this::getDossierTemplateStats, Collectors.toList()))); + List dossierTemplateStatsList = dictionarySummaryMap.entrySet().stream() .map(e -> - e.getValue().size() == 1 && e.getValue().get(0).getId() == null? - new DossierTemplateStats(e.getKey(), 0, new ArrayList<>()): - new DossierTemplateStats(e.getKey(), e.getValue().size(), e.getValue())) + e.getValue().size() == 1 && e.getValue().get(0).getId() == null ? + new DossierTemplateDictionaryStats(e.getKey(), 0, new ArrayList<>()) : + new DossierTemplateDictionaryStats(e.getKey(), e.getValue().size(), e.getValue())) .collect(Collectors.toList()); // add the dossier templates with no dictionaries but with dossiers - Set dossierTemplatesIdsFounded = dossierTemplateStatsList.stream().map(DossierTemplateStats::getDossierTemplateId).collect(Collectors.toSet()); + Set dossierTemplatesIdsFounded = dossierTemplateStatsList.stream().map(DossierTemplateDictionaryStats::getDossierTemplateId).collect(Collectors.toSet()); dossierTemplateIds.removeAll(dossierTemplatesIdsFounded); - List dossierTemplateStatsListForDossierNotFound = dossierTemplateIds.stream() - .map(e -> new DossierTemplateStats(e, 0, new ArrayList<>())).collect(Collectors.toList()); + List dossierTemplateStatsListForDossierNotFound = dossierTemplateIds.stream() + .map(e -> new DossierTemplateDictionaryStats(e, 0, new ArrayList<>())).collect(Collectors.toList()); dossierTemplateStatsList.addAll(dossierTemplateStatsListForDossierNotFound); return dossierTemplateStatsList; } @@ -47,4 +72,37 @@ public class DossierTemplateStatsService { private DictionarySummary getDossierTemplateStats(DictionarySummaryResponse response) { return new DictionarySummary(response.getId(), response.getType(), response.getName(), response.getEntriesCount()); } + + private DossierTemplateStats getDossierTemplateStats(String dossierTemplateId, String dossierTemplateName) { + + DossierTemplateStats dts = new DossierTemplateStats(); + + dts.setDossierTemplateId(dossierTemplateId); + dts.setName(dossierTemplateName); + dts.setNumberOfArchivedDossiers(dossierRepository.countArchived(dossierTemplateId)); + dts.setNumberOfActiveDossiers(dossierRepository.countActive(dossierTemplateId)); + dts.setNumberOfDeletedDossiers(dossierRepository.countSofDeleted(dossierTemplateId)); + dts.setNumberOfActiveFiles(fileRepository.countActiveFiles(dossierTemplateId)); + dts.setNumberOfActiveFiles(fileRepository.countSoftDeletedFiles(dossierTemplateId)); + + var processingCounts = fileRepository.countFilesByProcessingStatus(dossierTemplateId); + var workflowCounts = fileRepository.countFilesByWorkflowStatus(dossierTemplateId); + var pageCounts = fileRepository.countPages(dossierTemplateId); + + var dossierStatusCounts = dossierRepository.countByDossierStatus(dossierTemplateId); + + dts.setNumberOfPages(pageCounts.getNumberOfAnalyzedPages()); + dts.setNumberOfExcludedPages(pageCounts.getNumberOfExcludedPages()); + + dts.setFileCountPerProcessingStatus(processingCounts.stream() + .map(t -> new DossierTemplateStats.ProcessingStatusCount(t.getProcessingStatus(), t.getCount())).collect(Collectors.toList())); + dts.setFileCountPerWorkflowStatus(workflowCounts.stream() + .map(t -> new DossierTemplateStats.WorkflowStatusCount(t.getWorkflowStatus(), t.getCount())).collect(Collectors.toList())); + dts.setDossierCountByStatus(dossierStatusCounts.stream() + .map(t -> new DossierTemplateStats.DossierStatusCount(t.getDossierStatusId(), t.getCount())).collect(Collectors.toList())); + + dts.setDossiersInTemplate(dossierRepository.findActiveDossierIdsForTemplate(dossierTemplateId)); + + return dts; + } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 532d0fefd..026817b65 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -1,16 +1,13 @@ package com.iqser.red.service.peristence.v1.server.service; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.web.bind.annotation.RestController; - import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.MessageType; import com.iqser.red.service.search.v1.model.IndexMessageType; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.web.bind.annotation.RestController; @Slf4j @RestController @@ -86,20 +83,20 @@ public class FileStatusProcessingUpdateService { public void analysisFailed(String dossierId, String fileId) { - setStatusError(dossierId, fileId); + setStatusError(dossierId, fileId, "analysisFailed"); } public void ocrFailed(String dossierId, String fileId) { - setStatusError(dossierId, fileId); + setStatusError(dossierId, fileId, "ocrFailed"); } - private void setStatusError(String dossierId, String fileId) { + private void setStatusError(String dossierId, String fileId, String reason) { retryTemplate.execute(retryContext -> { - log.warn("Retrying {} time to set ERROR status for file {} in dossier {}", retryContext.getRetryCount(), fileId, dossierId); + log.warn("Retrying {} time to set ERROR status for file {} in dossier {} with reason {} ", retryContext.getRetryCount(), fileId, dossierId, reason); fileStatusService.setStatusError(fileId); return null; }); @@ -128,7 +125,7 @@ public class FileStatusProcessingUpdateService { public void indexingFailed(String dossierId, String fileId) { - setStatusError(dossierId, fileId); + setStatusError(dossierId, fileId, "indexingFailed"); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java index 579ee2ed2..c1ab54075 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTemplateTesterAndProvider.java @@ -31,10 +31,10 @@ public class DossierTemplateTesterAndProvider { @Autowired private LegalBasisClient legalBasisClient; - public DossierTemplate provideTestTemplate() { + public DossierTemplate provideTestTemplate(String name) { CreateOrUpdateDossierTemplateRequest cru = new CreateOrUpdateDossierTemplateRequest(); cru.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL)); - cru.setName("Template 1"); + cru.setName(name); cru.setDescription("Template 1"); cru.setRequestingUser("1"); cru.setValidFrom(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); @@ -42,7 +42,7 @@ public class DossierTemplateTesterAndProvider { DossierTemplate result = dossierTemplateClient.createOrUpdateDossierTemplate(cru); - assertThat(result.getName()).isEqualTo("Template 1"); + assertThat(result.getName()).isEqualTo(name); DossierTemplate loadedTemplate = dossierTemplateClient.getDossierTemplate(result.getId()); @@ -56,5 +56,9 @@ public class DossierTemplateTesterAndProvider { return loadedTemplate; } + public DossierTemplate provideTestTemplate() { + return provideTestTemplate("Template 1"); + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java index a24ff0a96..26eda785b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/DossierTesterAndProvider.java @@ -34,6 +34,19 @@ public class DossierTesterAndProvider { } public Dossier provideTestDossier(DossierTemplate testTemplate, String dossierName, DossierStatusInfo dossierStatus) { + + var result = provideTestDossierQuick(testTemplate,dossierName,dossierStatus); + + assertThat(result.getDossierName()).isEqualTo(dossierName); + + Dossier loadedDossier = dossierClient.getDossierById(result.getId(), false, false); + + assertThat(loadedDossier).isEqualTo(result); + + return loadedDossier; + } + + public Dossier provideTestDossierQuick(DossierTemplate testTemplate, String dossierName, DossierStatusInfo dossierStatus) { CreateOrUpdateDossierRequest cru = new CreateOrUpdateDossierRequest(); cru.setDownloadFileTypes(Sets.newHashSet(DownloadFileType.ORIGINAL)); cru.setDossierName(dossierName); @@ -52,13 +65,7 @@ public class DossierTesterAndProvider { Dossier result = dossierClient.addDossier(cru); - assertThat(result.getDossierName()).isEqualTo(dossierName); - - Dossier loadedDossier = dossierClient.getDossierById(result.getId(), false, false); - - assertThat(loadedDossier).isEqualTo(result); - - return loadedDossier; + return result; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index 6967ad213..aaab8505c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -48,11 +48,8 @@ public class FileTesterAndProvider { @SneakyThrows public FileModel testAndProvideFile(Dossier dossier, String fileName) { - - BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), fileName, dossier.getId(), "1"); - JSONPrimitive uploadResult = uploadClient.upload(upload); - - var file = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue()); + var fileId = testAndProvideFileQuick(dossier, fileName); + var file = fileClient.getFileStatus(dossier.getId(), fileId); assertThat(file.getId()).isNotBlank(); @@ -66,4 +63,15 @@ public class FileTesterAndProvider { return file; } + @SneakyThrows + public String testAndProvideFileQuick(Dossier dossier, String fileName) { + + + BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), fileName, dossier.getId(), "1"); + JSONPrimitive uploadResult = uploadClient.upload(upload); + + return uploadResult.getValue(); + + } + } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java index 3c9060d09..58d11215d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java @@ -1,6 +1,23 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; +import com.google.common.collect.Sets; +import com.iqser.red.service.peristence.v1.server.integration.client.*; +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateDictionaryStats; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; @@ -10,23 +27,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import com.google.common.collect.Sets; -import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient; -import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateStatsClient; -import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; -import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CreateOrUpdateDossierTemplateRequest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplate; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStats; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummary; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; +import static org.assertj.core.api.Assertions.assertThat; public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTest { private static final String TYPE_ID_1 = "type1"; @@ -36,6 +37,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe @Autowired private DossierTesterAndProvider dossierTesterAndProvider; + @Autowired + private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider; + @Autowired private DictionaryClient dictionaryClient; @@ -45,8 +49,25 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe @Autowired private DossierTemplateStatsClient dossierTemplateStatsClient; + @Autowired + private DossierClient dossierClient; + + @Autowired + private FileClient fileClient; + + @Autowired + private UploadClient uploadClient; + + @Autowired + private FileTesterAndProvider fileTesterAndProvider; + @Autowired + private FileStatusPersistenceService fileStatusPersistenceService; + + @Autowired + private DossierStatusClient dossierStatusClient; + @Test - public void testDossierTemplateStats() { + public void testDossierTemplateDictionaryStats() { var dossier = dossierTesterAndProvider.provideTestDossier(); @@ -155,9 +176,9 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe dossierTemplateIds.add(dossierTemplate4.getId()); dossierTemplateIds.add(dossierTemplate5.getId()); - List dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateStats(dossierTemplateIds); + List dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateDictionaryStats(dossierTemplateIds); assertThat(dossierTemplateStatsList.size()).isEqualTo(dossierTemplateIds.size()); - DossierTemplateStats dossierTemplateStats1 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate2.getId())).findAny().get(); + DossierTemplateDictionaryStats dossierTemplateStats1 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate2.getId())).findAny().get(); assertThat(dossierTemplateStats1.getDossierTemplateId()).isEqualTo(dossierTemplate2.getId()); assertThat(dossierTemplateStats1.getNumberOfDictionaries()).isEqualTo(2); @@ -166,7 +187,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe assertThat(dictionarySummaryList.stream().filter(d -> d.getType().equals(TYPE_ID_2)).findAny().get().getEntriesCount()).isEqualTo(entries2.size()); // dossier template with no type - DossierTemplateStats dossierTemplateStats3 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate4.getId())).findAny().get(); + DossierTemplateDictionaryStats dossierTemplateStats3 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate4.getId())).findAny().get(); assertThat(dossierTemplateStats3.getDossierTemplateId()).isEqualTo(dossierTemplate4.getId()); assertThat(dossierTemplateStats3.getNumberOfDictionaries()).isZero(); @@ -175,10 +196,10 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe entries22.add(entries2.get(2)); dictionaryClient.deleteEntries(addedType2.getTypeId(), entries22, DictionaryEntryType.ENTRY); - dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateStats(dossierTemplateIds); + dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateDictionaryStats(dossierTemplateIds); assertThat(dossierTemplateStatsList.size()).isEqualTo(dossierTemplateIds.size()); - DossierTemplateStats dossierTemplateStats2 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate2.getId())).findAny().get(); + DossierTemplateDictionaryStats dossierTemplateStats2 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate2.getId())).findAny().get(); dictionarySummaryList = dossierTemplateStats2.getDictionarySummaryList(); assertThat(dictionarySummaryList.stream().filter(d -> d.getType().equals(TYPE_ID_2)).findAny().get().getEntriesCount()).isEqualTo(entries2.size() - entries22.size()); @@ -192,10 +213,10 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe var entries23loaded = dictionaryClient.getEntriesForType(addedType2.getTypeId(), null, DictionaryEntryType.ENTRY); assertThat(entries23loaded.stream().filter(e -> !e.isDeleted()).collect(Collectors.toList())).isEmpty(); - dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateStats(dossierTemplateIds); + dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateDictionaryStats(dossierTemplateIds); assertThat(dossierTemplateStatsList.size()).isEqualTo(dossierTemplateIds.size()); - DossierTemplateStats dossierTemplateStats23 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate2.getId())).findAny().get(); + DossierTemplateDictionaryStats dossierTemplateStats23 = dossierTemplateStatsList.stream().filter(d -> d.getDossierTemplateId().equals(dossierTemplate2.getId())).findAny().get(); dictionarySummaryList = dossierTemplateStats23.getDictionarySummaryList(); assertThat(dictionarySummaryList.stream().filter(d -> d.getType().equals(TYPE_ID_2)).findAny().isEmpty()).isTrue(); @@ -214,4 +235,58 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe return dossierTemplateClient.createOrUpdateDossierTemplate(cru); } + + @Test + public void testDossierTemplateStats() { + + for (int i = 0; i < 2; i++) { + var template = dossierTemplateTesterAndProvider.provideTestTemplate("test template: " + i); + + var status = dossierStatusClient.createOrUpdateDossierStatus(CreateOrUpdateDossierStatusRequest.builder() + .dossierTemplateId(template.getId()).name("test").rank(100).build()); + for (int j = 0; j < 8; j++) { + + var dossier = dossierTesterAndProvider.provideTestDossierQuick(template, "test dossier: " + j + " - " + i, j % 2 == 0 ? status : null); + + for (int k = 0; k < 12; k++) { + var fileId = fileTesterAndProvider.testAndProvideFileQuick(dossier, "file: " + k); + + if (k % 2 == 0) { + fileStatusPersistenceService.updateProcessingStatus(fileId, k, + 0L, 0L, 0L, 0L, 0L, 1, 1); + fileClient.excludePages(dossier.getId(), fileId, Set.of(k)); + } + if (k % 3 == 0) { + uploadClient.deleteFile(dossier.getId(), fileId); + } + if (j % 5 == 0) { + uploadClient.hardDeleteFiles(dossier.getId(), Set.of(fileId)); + } + if (j % 7 == 0) { + fileClient.setStatusUnderReview(dossier.getId(), fileId, "x"); + } + if (j % 11 == 0) { + fileClient.setStatusUnderApproval(dossier.getId(), fileId, "x"); + } + } + + if (j % 3 == 0) { + dossierClient.archiveDossiers(Set.of(dossier.getId())); + } + if (j % 5 == 0) { + dossierClient.delete(dossier.getId()); + } + if (j % 7 == 0) { + dossierClient.hardDeleteDossiers(Set.of(dossier.getId())); + } + + } + } + + long t1 = System.currentTimeMillis(); + var stats = dossierTemplateStatsClient.getDossierTemplateStats(); + assertThat(stats.size()).isEqualTo(2); + long t2 = System.currentTimeMillis(); + System.out.println("Time for Stats: " + (t2 - t1) + "ms"); + } }