Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'feature/release/2.465.x-dossier-stats-backport' into 'release/2.465.x'

Browse Source 
RED-9782: Enabled to upload file and import only imported redactions with...

See merge request redactmanager/persistence-service!834
This commit is contained in:
Timo Bejan 2024-11-07 08:26:43 +01:00
commit 441cebdd1b
10 changed files with 363 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/projection/DossierStatsFileProjection.java Normal file
View File

@ -0,0 +1,44 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.projection;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.Set;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import jakarta.persistence.Convert;
public interface DossierStatsFileProjection {
boolean isHasRedactions();
boolean isHasHints();
boolean isHasSuggestions();
boolean isHasUpdates();
ProcessingStatus getProcessingStatus();
OffsetDateTime getLastUpdated();
OffsetDateTime getFileManipulationDate();
WorkflowStatus getWorkflowStatus();
int getNumberOfPages();
@Convert(converter = JSONIntegerSetConverter.class)
Set<Integer> getExcludedPages();
}

104
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierStatsService.java
View File

@ -2,17 +2,17 @@ package com.iqser.red.service.persistence.management.v1.processor.service;
import static com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException.DOSSIER_NOT_FOUND_MESSAGE;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStats;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -52,64 +52,58 @@ public class DossierStatsService {
private DossierStats computeDossierStats(DossierEntity dossierEntity) {
var dossierId = dossierEntity.getId();
DossierStats dossierStats = new DossierStats();
String dossierId = dossierEntity.getId();
if (dossierEntity.getHardDeletedTime() != null) {
throw new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE);
}
dossierStats.setDossierId(dossierId);
// get the associated files
List<FileModel> files = fileStatusService.getActiveFiles(dossierId);
dossierStats.setNumberOfFiles(files.size());
dossierStats.setNumberOfSoftDeletedFiles(fileStatusService.countSoftDeletedFiles(dossierId));
dossierStats.setNumberOfPages(files.stream()
.mapToInt(FileModel::getNumberOfPages).sum());
dossierStats.setNumberOfExcludedPages(files.stream()
.mapToInt(f -> f.getExcludedPages().size()).sum());
files.stream()
.filter(FileModel::isHasRedactions)
.findAny()
.ifPresent(v -> dossierStats.setHasRedactionsFilePresent(true));
files.stream()
.filter(FileModel::isHasHints)
.filter(f -> !f.isHasRedactions())
.findAny()
.ifPresent(v -> dossierStats.setHasHintsNoRedactionsFilePresent(true));
files.stream()
.filter(FileModel::isHasSuggestions)
.findAny()
.ifPresent(v -> dossierStats.setHasSuggestionsFilePresent(true));
files.stream()
.filter(FileModel::isHasUpdates)
.findAny()
.ifPresent(v -> dossierStats.setHasUpdatesFilePresent(true));
files.stream()
.filter(f -> !f.isHasRedactions())
.filter(f -> !f.isHasHints())
.filter(f -> !f.isHasSuggestions())
.filter(f -> !f.isHasUpdates())
.findAny()
.ifPresent(v -> dossierStats.setHasNoFlagsFilePresent(true));
var fileCountPerProcessingStatus = files.stream()
.collect(Collectors.toMap(FileModel::getProcessingStatus, e -> 1, Math::addExact));
dossierStats.setFileCountPerProcessingStatus(fileCountPerProcessingStatus);
List<DossierStatsFileProjection> files = fileStatusService.getDossierStatsFiles(dossierId);
int numberOfSoftDeletedFiles = fileStatusService.countSoftDeletedFiles(dossierId);
var fileCountPerWorkflowStatus = files.stream()
.collect(Collectors.toMap(FileModel::getWorkflowStatus, e -> 1, Math::addExact));
dossierStats.setFileCountPerWorkflowStatus(fileCountPerWorkflowStatus);
DossierStats stats = DossierStats.builder()
.dossierId(dossierId)
.numberOfFiles(files.size())
.numberOfSoftDeletedFiles(numberOfSoftDeletedFiles)
.fileCountPerProcessingStatus(new EnumMap<>(ProcessingStatus.class))
.fileCountPerWorkflowStatus(new EnumMap<>(WorkflowStatus.class))
.build();
files.stream()
.sorted(Comparator.comparing(FileModel::getLastUpdated, Comparator.nullsLast(Comparator.reverseOrder())))
.findFirst()
.ifPresent(file -> dossierStats.setLastFileUpdateDate(file.getLastUpdated()));
files.stream()
.sorted(Comparator.comparing(FileModel::getFileManipulationDate, Comparator.nullsLast(Comparator.reverseOrder())))
.findFirst()
.ifPresent(file -> dossierStats.setFileManipulationDate(file.getFileManipulationDate()));
return dossierStats;
for (DossierStatsFileProjection file : files) {
stats.setNumberOfPages(stats.getNumberOfPages() + file.getNumberOfPages());
stats.setNumberOfExcludedPages(stats.getNumberOfExcludedPages() + file.getExcludedPages().size());
if (file.isHasRedactions()) {
stats.setHasRedactionsFilePresent(true);
}
if (file.isHasHints() && !file.isHasRedactions()) {
stats.setHasHintsNoRedactionsFilePresent(true);
}
if (file.isHasSuggestions()) {
stats.setHasSuggestionsFilePresent(true);
}
if (file.isHasUpdates()) {
stats.setHasUpdatesFilePresent(true);
}
if (!file.isHasRedactions() && !file.isHasHints() && !file.isHasSuggestions() && !file.isHasUpdates()) {
stats.setHasNoFlagsFilePresent(true);
}
stats.getFileCountPerProcessingStatus().merge(file.getProcessingStatus(), 1, Integer::sum);
stats.getFileCountPerWorkflowStatus().merge(file.getWorkflowStatus(), 1, Integer::sum);
if (stats.getLastFileUpdateDate() == null ||
(file.getLastUpdated() != null && file.getLastUpdated().isAfter(stats.getLastFileUpdateDate()))) {
stats.setLastFileUpdateDate(file.getLastUpdated());
}
if (stats.getFileManipulationDate() == null ||
(file.getFileManipulationDate() != null && file.getFileManipulationDate().isAfter(stats.getFileManipulationDate()))) {
stats.setFileManipulationDate(file.getFileManipulationDate());
}
}
return stats;
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java
View File

@ -10,6 +10,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ApplicationType;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@ -132,6 +133,11 @@ public class FileStatusService {
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
}
public List<DossierStatsFileProjection> getDossierStatsFiles(String dossierId) {
return fileStatusPersistenceService.getFilesForDossierStats(dossierId);
}
@Transactional
public List<FileModel> getDossierStatus(String dossierId) {

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -11,6 +11,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@ -428,6 +429,11 @@ public class FileStatusPersistenceService {
.collect(Collectors.toList());
}
public List<DossierStatsFileProjection> getFilesForDossierStats(String dossierId) {
return fileRepository.findDossierStatsProjectionFileProjectionByDossierId(dossierId);
}
public List<FileEntity> getAllRelevantErrorFiles() {

4
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -10,6 +10,7 @@ import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.projection.DossierStatsFileProjection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection;
@ -412,6 +413,9 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
+ "where f.id in (:fileIds)")
int hardDeleteFiles(@Param("fileIds") List<String> fileIds, @Param("processingStatus") ProcessingStatus processingStatus, @Param("deletionTime") OffsetDateTime deletionTime);
@Query("SELECT f FROM FileEntity f WHERE f.dossierId = :dossierId AND f.hardDeletedTime IS NULL AND f.deleted IS NULL")
List<DossierStatsFileProjection> findDossierStatsProjectionFileProjectionByDossierId(@Param("dossierId") String dossierId);
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/db.changelog-tenant.yaml
View File

@ -220,4 +220,8 @@ databaseChangeLog:
- include:
file: db/changelog/tenant/143-modify-download-redaction-file-status-details.yaml
- include:
file: db/changelog/tenant/144-add-error-code-to-file.yaml
file: db/changelog/tenant/144-add-error-code-to-file.yaml
- include:
file: db/changelog/tenant/145-add-indexes-to-file-table.yaml
- include:
file: db/changelog/tenant/149-add-indexes-across-tables-for-performance.yaml

25
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/145-add-indexes-to-file-table.yaml Normal file
View File

@ -0,0 +1,25 @@
databaseChangeLog:
- changeSet:
id: add-indexes-to-file-table
author: timo
changes:
- createIndex:
columns:
- column:
name: dossier_id
- column:
name: last_updated
indexName: file_dossier_id_last_updated_idx
unique: false
tableName: file
- createIndex:
columns:
- column:
name: dossier_id
- column:
name: deleted
- column:
name: hard_deleted_time
indexName: file_dossier_id_deleted_hard_deleted_time_idx
unique: false
tableName: file

220
persistence-service-v1/persistence-service-processor-v1/src/main/resources/db/changelog/tenant/149-add-indexes-across-tables-for-performance.yaml Normal file
View File

@ -0,0 +1,220 @@
databaseChangeLog:
- changeSet:
id: create_index_if_not_exists_idx_file_last_updated
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_file_last_updated
tableName: file
changes:
- createIndex:
tableName: file
indexName: idx_file_last_updated
columns:
- column:
name: last_updated
- changeSet:
id: create_index_if_not_exists_idx_file_deleted_hard_deleted_time
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_file_deleted_hard_deleted_time
tableName: file
changes:
- createIndex:
tableName: file
indexName: idx_file_deleted_hard_deleted_time
columns:
- column:
name: deleted
- column:
name: hard_deleted_time
- changeSet:
id: create_index_if_not_exists_idx_file_dossier_id_deleted_hard_deleted_time
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_file_dossier_id_deleted_hard_deleted_time
tableName: file
changes:
- createIndex:
tableName: file
indexName: idx_file_dossier_id_deleted_hard_deleted_time
columns:
- column:
name: dossier_id
- column:
name: deleted
- column:
name: hard_deleted_time
- changeSet:
id: create_index_if_not_exists_idx_dossier_last_updated
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_dossier_last_updated
tableName: dossier
changes:
- createIndex:
tableName: dossier
indexName: idx_dossier_last_updated
columns:
- column:
name: last_updated
- changeSet:
id: create_index_if_not_exists_idx_dossier_soft_deleted_time_hard_deleted_time
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_dossier_soft_deleted_time_hard_deleted_time
tableName: dossier
changes:
- createIndex:
tableName: dossier
indexName: idx_dossier_soft_deleted_time_hard_deleted_time
columns:
- column:
name: soft_deleted_time
- column:
name: hard_deleted_time
- changeSet:
id: create_index_if_not_exists_idx_dossier_id_soft_deleted_time_hard_deleted_time
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_dossier_id_soft_deleted_time_hard_deleted_time
tableName: dossier
changes:
- createIndex:
tableName: dossier
indexName: idx_dossier_id_soft_deleted_time_hard_deleted_time
columns:
- column:
name: id
- column:
name: soft_deleted_time
- column:
name: hard_deleted_time
- changeSet:
id: create_index_if_not_exists_idx_dossier_soft_deleted_time_hard_deleted_time_archived_time
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_dossier_soft_deleted_time_hard_deleted_time_archived_time
tableName: dossier
changes:
- createIndex:
tableName: dossier
indexName: idx_dossier_soft_deleted_time_hard_deleted_time_archived_time
columns:
- column:
name: soft_deleted_time
- column:
name: hard_deleted_time
- column:
name: archived_time
- changeSet:
id: create_index_if_not_exists_idx_dossier_dossier_template_id_soft_deleted_time_hard_deleted_time_archived_time
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_dossier_dossier_template_id_soft_deleted_time_hard_deleted_time_archived_time
tableName: dossier
changes:
- createIndex:
tableName: dossier
indexName: idx_dossier_dossier_template_id_soft_deleted_time_hard_deleted_time_archived_time
columns:
- column:
name: dossier_template_id
- column:
name: soft_deleted_time
- column:
name: hard_deleted_time
- column:
name: archived_time
- changeSet:
id: create_index_if_not_exists_idx_notification_preference_user_id_in_app_notifications_enabled
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_notification_preference_user_id_in_app_notifications_enabled
tableName: notification_preference
changes:
- createIndex:
tableName: notification_preference
indexName: idx_notification_preference_user_id_in_app_notifications_enabled
columns:
- column:
name: user_id
- column:
name: in_app_notifications_enabled
- changeSet:
id: create_index_if_not_exists_idx_notification_user_id_creation_date_soft_deleted
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_notification_user_id_creation_date_soft_deleted
tableName: notification
changes:
- createIndex:
tableName: notification
indexName: idx_notification_user_id_creation_date_soft_deleted
columns:
- column:
name: user_id
- column:
name: creation_date
- column:
name: soft_deleted
- changeSet:
id: create_index_if_not_exists_idx_entity_dossier_template_id
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_entity_dossier_template_id
tableName: entity
changes:
- createIndex:
tableName: entity
indexName: idx_entity_dossier_template_id
columns:
- column:
name: dossier_template_id
- changeSet:
id: create_index_if_not_exists_idx_entity_dossier_id
author: Timo
preConditions:
- not:
indexExists:
indexName: idx_entity_dossier_id
tableName: entity
changes:
- createIndex:
tableName: entity
indexName: idx_entity_dossier_id
columns:
- column:
name: dossier_id

2
persistence-service-v1/persistence-service-server-v1/src/test/resources/testcontainers.properties
View File

@ -1 +1 @@
hub.image.name.prefix=docker-dev.knecon.com/tests/
hub.image.name.prefix=docker-dev.knecon.com/tests/

5
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/DossierStats.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model;
import java.io.Serializable;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Map;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
@ -41,9 +42,9 @@ public class DossierStats implements Serializable {
@Schema(description = "True if at least one file in the dossier has none of the other flags")
private boolean hasNoFlagsFilePresent; // true if at least one file in the dossier has none of the other flags
@Schema(description = "Number of files in each processing status")
private Map<ProcessingStatus, Integer> fileCountPerProcessingStatus;
private Map<ProcessingStatus, Integer> fileCountPerProcessingStatus = new HashMap<>();
@Schema(description = "Number of files in each status")
private Map<WorkflowStatus, Integer> fileCountPerWorkflowStatus;
private Map<WorkflowStatus, Integer> fileCountPerWorkflowStatus = new HashMap<>();
@Schema(description = "The most recent file modification date")
private OffsetDateTime lastFileUpdateDate;
@Schema(description = "The most recent file manipulation date")