From 2ef28d8ecb9cf35b7bdc63a4c79301b92f2bae2c Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Thu, 21 Sep 2023 17:22:41 +0300 Subject: [PATCH 1/3] RED-7326 - backport license storage implementation --- .../v1/api/model/license/LicenseReport.java | 21 +- .../api/model/license/MonthlyReportData.java | 26 ++ .../api/resources/LicenseReportResource.java | 4 +- .../FileStatusPersistenceService.java | 4 +- .../repository/FileRepository.java | 2 +- .../controller/LicenseReportController.java | 6 +- .../v1/server/service/FileStatusService.java | 10 +- .../server/service/LicenseReportService.java | 271 +++++++++++------- .../integration/tests/LicenseReportTest.java | 3 +- 9 files changed, 215 insertions(+), 132 deletions(-) create mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/MonthlyReportData.java diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/LicenseReport.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/LicenseReport.java index 3ff6f2e3f..4eb7444de 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/LicenseReport.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/LicenseReport.java @@ -15,28 +15,21 @@ import lombok.NoArgsConstructor; @NoArgsConstructor public class LicenseReport { + private long totalFilesUploadedBytes; + private long activeFilesUploadedBytes; + private long trashFilesUploadedBytes; + private long archivedFilesUploadedBytes; private int numberOfAnalyzedFiles; private int numberOfOcrFiles; - private int numberOfDossiers; - private int numberOfAnalyzedPages; + private long analysedFilesBytes; private int numberOfOcrPages; - - private int numberOfAnalyses; // includes reanalysis counts - private Instant startDate; - private Instant endDate; - private int offset; - - private int limit; - - private List data = new ArrayList<>(); - - // To be used for consecutive/paged calls - private String requestId; + @Builder.Default + private List monthlyData = new ArrayList<>(); } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/MonthlyReportData.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/MonthlyReportData.java new file mode 100644 index 000000000..a30831c3c --- /dev/null +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/license/MonthlyReportData.java @@ -0,0 +1,26 @@ +package com.iqser.red.service.persistence.service.v1.api.model.license; + +import java.time.Instant; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class MonthlyReportData { + + private Instant startDate; + private Instant endDate; + private long totalFilesUploadedBytes; + private long activeFilesUploadedBytes; + private long trashFilesUploadedBytes; + private long archivedFilesUploadedBytes; + private int numberOfAnalyzedPages; + private long analysedFilesBytes; + private int numberOfOcrPages; + +} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/LicenseReportResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/LicenseReportResource.java index c3b19a6b7..7e9498226 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/LicenseReportResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/LicenseReportResource.java @@ -16,8 +16,6 @@ public interface LicenseReportResource { @ResponseBody @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = "/report/license", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - LicenseReport getLicenseReport(@RequestBody LicenseReportRequest licenseReportRequest, - @RequestParam(value = "offset", defaultValue = "0") int offset, - @RequestParam(value = "limit", defaultValue = "20") int limit); + LicenseReport getLicenseReport(@RequestBody LicenseReportRequest licenseReportRequest); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index f8a240616..d7074cbcd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -290,9 +290,9 @@ public class FileStatusPersistenceService { } - public List getStatusesForDossiersAndTimePeriod(Set dossierIds, OffsetDateTime start, OffsetDateTime end) { + public List getStatusesAddedBefore(OffsetDateTime end) { - return fileRepository.findByDossierIdInAndAddedBetween(dossierIds, start, end); + return fileRepository.findByAddedBefore(end); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 52577bf5e..1d66ca871 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -25,7 +25,7 @@ public interface FileRepository extends JpaRepository { List findByDossierId(String dossierId); - List findByDossierIdInAndAddedBetween(Set dossierIds, OffsetDateTime start, OffsetDateTime end); + List findByAddedBefore(OffsetDateTime end); @Modifying diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/LicenseReportController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/LicenseReportController.java index 9de2e407b..5c71e53d2 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/LicenseReportController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/LicenseReportController.java @@ -19,11 +19,9 @@ public class LicenseReportController implements LicenseReportResource { @Override - public LicenseReport getLicenseReport(@RequestBody LicenseReportRequest licenseReportRequest, - @RequestParam(value = "offset", defaultValue = "0") int offset, - @RequestParam(value = "limit", defaultValue = "20") int limit) { + public LicenseReport getLicenseReport(@RequestBody LicenseReportRequest licenseReportRequest) { - return licenseReportService.getLicenseReport(licenseReportRequest, offset, limit); + return licenseReportService.getLicenseReport(licenseReportRequest); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index d3cc5c88b..08730e918 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -40,6 +40,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -103,12 +104,11 @@ public class FileStatusService { return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); } + @Transactional + public List getStatusesAddedBefore(OffsetDateTime end) { - public List getStatusesForDossiersAndTimePeriod(Set dossierIds, OffsetDateTime start, OffsetDateTime end) { - - var fileEntities = fileStatusPersistenceService.getStatusesForDossiersAndTimePeriod(dossierIds, start, end); - var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); - return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); + var fileEntities = fileStatusPersistenceService.getStatusesAddedBefore(end); + return MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper()); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java index b50047ed5..8a09b5c33 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java @@ -1,27 +1,27 @@ package com.iqser.red.service.peristence.v1.server.service; -import java.time.Duration; -import java.time.Instant; import java.time.OffsetDateTime; +import java.time.YearMonth; import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; -import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.license.LicenseReport; import com.iqser.red.service.persistence.service.v1.api.model.license.LicenseReportRequest; -import com.iqser.red.service.persistence.service.v1.api.model.license.ReportData; +import com.iqser.red.service.persistence.service.v1.api.model.license.MonthlyReportData; -import lombok.AllArgsConstructor; -import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -34,107 +34,176 @@ public class LicenseReportService { private final FileStatusService fileStatusService; private final DossierService dossierService; + public LicenseReport getLicenseReport(LicenseReportRequest licenseReportRequest) { - public LicenseReport getLicenseReport(LicenseReportRequest licenseReportRequest, int offset, int limit) { - - log.info("Generating licence-report"); - - Instant start = null; - if (log.isInfoEnabled()) { - start = Instant.now(); + if (licenseReportRequest.getEndDate().isBefore(licenseReportRequest.getStartDate())) { + throw new BadRequestException("Invalid date period: End date is before start date."); } - if (StringUtils.isEmpty(licenseReportRequest.getRequestId())) { - licenseReportRequest.setRequestId(UUID.randomUUID().toString()); + var files = fileStatusService.getStatusesAddedBefore(OffsetDateTime.ofInstant(licenseReportRequest.getEndDate(), UTC_ZONE_ID)); + var addDossiers = dossierService.getAllDossiers(); + + files.sort(Comparator.comparing(FileModel::getAdded)); + + var dossiersById = addDossiers.stream().collect(Collectors.toMap(DossierEntity::getId, Function.identity())); + + Map> adds = new HashMap<>(); + Map> softDeletes = new HashMap<>(); + Map> archives = new HashMap<>(); + Map> hardDeletes = new HashMap<>(); + Map> ocrs = new HashMap<>(); + + + for (var file : files) { + + adds.computeIfAbsent(YearMonth.from(file.getAdded()), entry -> new HashSet<>()).add(file); + + if (file.getDeleted() != null && file.getDeleted().toInstant().isBefore(licenseReportRequest.getEndDate())) { + softDeletes.computeIfAbsent(YearMonth.from(file.getDeleted()), entry -> new HashSet<>()).add(file); + } + if(dossiersById.get(file.getDossierId()).getSoftDeletedTime() != null && dossiersById.get(file.getDossierId()).getSoftDeletedTime().toInstant().isBefore(licenseReportRequest.getEndDate())){ + softDeletes.computeIfAbsent(YearMonth.from(dossiersById.get(file.getDossierId()).getSoftDeletedTime()), entry -> new HashSet<>()).add(file); + } + if (file.getHardDeletedTime() != null && file.getHardDeletedTime().toInstant().isBefore(licenseReportRequest.getEndDate())) { + hardDeletes.computeIfAbsent(YearMonth.from(file.getHardDeletedTime()), entry -> new HashSet<>()).add(file); + } + if(dossiersById.get(file.getDossierId()).getHardDeletedTime() != null && dossiersById.get(file.getDossierId()).getHardDeletedTime().toInstant().isBefore(licenseReportRequest.getEndDate())){ + hardDeletes.computeIfAbsent(YearMonth.from(dossiersById.get(file.getDossierId()).getHardDeletedTime()), entry -> new HashSet<>()).add(file); + } + if (dossiersById.get(file.getDossierId()).getArchivedTime() != null && dossiersById.get(file.getDossierId()) + .getArchivedTime() + .toInstant() + .isBefore(licenseReportRequest.getEndDate())) { + archives.computeIfAbsent(YearMonth.from(dossiersById.get(file.getDossierId()).getArchivedTime()), entry -> new HashSet<>()).add(file); + } + if (file.getOcrStartTime() != null && file.getOcrStartTime().toInstant().isBefore(licenseReportRequest.getEndDate())) { + ocrs.computeIfAbsent(YearMonth.from(file.getOcrStartTime()), entry -> new HashSet<>()).add(file); + } } - DetailedReportData detailedReportData = loadDetailedReportData(licenseReportRequest); + YearMonth currentMonth = !files.isEmpty() && files.get(0).getAdded().toInstant().isBefore(licenseReportRequest.getStartDate()) ? YearMonth.from(files.get(0) + .getAdded()) : YearMonth.from(licenseReportRequest.getStartDate().atZone(UTC_ZONE_ID).toLocalDate()); + YearMonth endMonth = YearMonth.from(licenseReportRequest.getEndDate().atZone(UTC_ZONE_ID).toLocalDate()); + YearMonth reportStartMonth = YearMonth.from(licenseReportRequest.getStartDate().atZone(UTC_ZONE_ID).toLocalDate()); - LicenseReport licenseReport = new LicenseReport(); - licenseReport.setNumberOfAnalyzedPages(detailedReportData.getTotalPagesAnalyzed()); - licenseReport.setNumberOfAnalyzedFiles(detailedReportData.getData().size()); - licenseReport.setNumberOfOcrFiles((int) detailedReportData.getData().stream().filter(reportData -> reportData.getNumberOfOcrPages() > 0).count()); - licenseReport.setNumberOfOcrPages(detailedReportData.getTotalOcrPages()); - licenseReport.setNumberOfDossiers(detailedReportData.getNumberOfDossiers()); - licenseReport.setNumberOfAnalyses(detailedReportData.getTotalNumberOfAnalyses()); - licenseReport.setData(detailedReportData.getData().subList(offset, Math.min(offset + limit, detailedReportData.getData().size()))); - licenseReport.setOffset(offset); - licenseReport.setLimit(limit); - licenseReport.setStartDate(detailedReportData.getStartDate()); - licenseReport.setEndDate(detailedReportData.getEndDate()); - licenseReport.setRequestId(licenseReportRequest.getRequestId()); + List monthlyData = new ArrayList<>(); - if (start != null) { - log.info("getLicenceReport took {} to process", Duration.between(start, Instant.now()).toString()); + // This values are what is in system at that point. Including all added values before the period. + long activeFilesUploadedBytes = 0; + long trashFilesUploadedBytes = 0; + long archivedFilesUploadedBytes = 0; + + // This values are not what is currently in the system at that point. They are what is added in the period. + int numberOfAnalyzedPages = 0; + int numberOfOcrPages = 0; + int numberOfAnalyzedFiles = 0; + long analysedFilesBytes = 0; + int numberOfOcrFiles = 0; + + while (!currentMonth.isAfter(endMonth)) { + + int currentMonthNumberOfAnalyzedPages = 0; + int currentanalysedFilesBytes = 0; + int currentMonthNumberOfOcrPages = 0; + + var addedFilesInMonth = adds.get(currentMonth); + if (addedFilesInMonth != null) { + for (var add : addedFilesInMonth) { + activeFilesUploadedBytes += add.getFileSize(); + + if (add.getAdded().toInstant().isAfter(licenseReportRequest.getStartDate())) { + + numberOfAnalyzedPages += add.getNumberOfPages(); + currentMonthNumberOfAnalyzedPages += add.getNumberOfPages(); + numberOfAnalyzedFiles++; + analysedFilesBytes += add.getFileSize(); + currentanalysedFilesBytes += add.getFileSize(); + } + } + } + + var softDeletedFilesInMonth = softDeletes.get(currentMonth); + if (softDeletedFilesInMonth != null) { + for (var softDeleted : softDeletedFilesInMonth) { + if (dossiersById.get(softDeleted.getDossierId()).getArchivedTime() != null) { + archivedFilesUploadedBytes -= softDeleted.getFileSize(); + } else { + activeFilesUploadedBytes -= softDeleted.getFileSize(); + } + trashFilesUploadedBytes += softDeleted.getFileSize(); + } + } + + var archivedFilesInMonth = archives.get(currentMonth); + if (archivedFilesInMonth != null) { + for (var archived : archivedFilesInMonth) { + activeFilesUploadedBytes -= archived.getFileSize(); + archivedFilesUploadedBytes += archived.getFileSize(); + } + } + + var hardDeletedFilesInMonth = hardDeletes.get(currentMonth); + if (hardDeletedFilesInMonth != null) { + for (var hardDeleted : hardDeletedFilesInMonth) { + if (hardDeleted.getDeleted() != null || dossiersById.get(hardDeleted.getDossierId()).getSoftDeletedTime() != null) { + trashFilesUploadedBytes -= hardDeleted.getFileSize(); + } else { + activeFilesUploadedBytes -= hardDeleted.getFileSize(); + } + } + } + + var ocrFilesInMonth = ocrs.get(currentMonth); + if (ocrFilesInMonth != null) { + for (var ocrFile : ocrFilesInMonth) { + + if (ocrFile.getOcrStartTime().toInstant().isAfter(licenseReportRequest.getStartDate())) { + numberOfOcrPages += ocrFile.getNumberOfPages(); // We count the entire document if ocr is performed. + currentMonthNumberOfOcrPages += ocrFile.getNumberOfPages(); + numberOfOcrFiles++; + } + } + } + + if (currentMonth.equals(reportStartMonth) || currentMonth.isAfter(reportStartMonth)) { + + var monthEndDate = currentMonth.atEndOfMonth().atTime(23, 59).atZone(UTC_ZONE_ID).toInstant(); + + monthlyData.add(MonthlyReportData.builder() + .startDate(currentMonth.atDay(1).atStartOfDay(UTC_ZONE_ID).toInstant()) + .endDate(monthEndDate.isBefore(licenseReportRequest.getEndDate()) ? monthEndDate : licenseReportRequest.getEndDate()) + .activeFilesUploadedBytes(activeFilesUploadedBytes) + .trashFilesUploadedBytes(trashFilesUploadedBytes) + .archivedFilesUploadedBytes(archivedFilesUploadedBytes) + .totalFilesUploadedBytes(activeFilesUploadedBytes + trashFilesUploadedBytes + archivedFilesUploadedBytes) + .numberOfAnalyzedPages(currentMonthNumberOfAnalyzedPages) + .numberOfOcrPages(currentMonthNumberOfOcrPages) + .analysedFilesBytes(currentanalysedFilesBytes) + .build()); + } + + currentMonth = currentMonth.plusMonths(1); } - return licenseReport; - - } - - - private DetailedReportData loadDetailedReportData(LicenseReportRequest licenseReportRequest) { - - log.debug("No licence-report found in cache, generating new report"); - Instant start = null; - if (log.isInfoEnabled()) { - start = Instant.now(); - } - - final Set dossierIds; - if (CollectionUtils.isEmpty(licenseReportRequest.getDossierIds())) { - dossierIds = dossierService.getAllDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet()); - } else { - dossierIds = new HashSet<>(licenseReportRequest.getDossierIds()); - } - - var result = new DetailedReportData(new LinkedList<>(), dossierIds.size(), 0, 0, 0, licenseReportRequest.getStartDate(), licenseReportRequest.getEndDate()); - - fileStatusService.getStatusesForDossiersAndTimePeriod( // - dossierIds, // - OffsetDateTime.ofInstant(licenseReportRequest.getStartDate(), UTC_ZONE_ID), // - OffsetDateTime.ofInstant(licenseReportRequest.getEndDate(), UTC_ZONE_ID)) // - .forEach(fileStatus -> { - ReportData reportData = new ReportData(); - reportData.setDossier(fileStatus.getDossierId()); - reportData.setFileName(fileStatus.getFilename()); - reportData.setAddedDate(fileStatus.getAdded().toInstant()); - reportData.setLastUpdatedDate(fileStatus.getLastUpdated() == null ? null : fileStatus.getLastUpdated().toInstant()); - reportData.setDeletedDate(fileStatus.getDeleted() == null ? null : fileStatus.getDeleted().toInstant()); - reportData.setWorkflowStatus(fileStatus.getWorkflowStatus()); - reportData.setNumberOfAnalyzedPages(fileStatus.getNumberOfPages()); - reportData.setNumberOfOcrPages(fileStatus.getOcrStartTime() != null ? fileStatus.getNumberOfPages() : 0); - reportData.setAnalysisCount(fileStatus.getNumberOfAnalyses()); - result.totalPagesAnalyzed += fileStatus.getNumberOfPages(); - result.totalOcrPages += fileStatus.getOcrStartTime() != null ? fileStatus.getNumberOfPages() : 0; - result.totalNumberOfAnalyses += fileStatus.getNumberOfAnalyses(); - result.data.add(reportData); - }); - - if (start != null) { - log.info("loadReport took {} to process", Duration.between(start, Instant.now()).toString()); - } - - return result; - - } - - - // This was also used to cache results so that subsequent pagination works faster. - // Currently, pagination is unused, and it is unclear, if caching is needed. - // This intermediate object may be re-used for caching and pagination, if needed. - @Data - @AllArgsConstructor - private static class DetailedReportData { - - List data; - int numberOfDossiers; - int totalNumberOfAnalyses; - int totalPagesAnalyzed; - int totalOcrPages; - Instant startDate; - Instant endDate; - + return LicenseReport.builder() + .totalFilesUploadedBytes(activeFilesUploadedBytes + trashFilesUploadedBytes + archivedFilesUploadedBytes) + .activeFilesUploadedBytes(activeFilesUploadedBytes) + .trashFilesUploadedBytes(trashFilesUploadedBytes) + .archivedFilesUploadedBytes(archivedFilesUploadedBytes) + .numberOfAnalyzedPages(numberOfAnalyzedPages) + .numberOfOcrPages(numberOfOcrPages) + .numberOfAnalyzedFiles(numberOfAnalyzedFiles) + .analysedFilesBytes(analysedFilesBytes) + .numberOfOcrFiles(numberOfOcrFiles) + .numberOfDossiers(addDossiers.stream().filter(dossier -> // + dossier.getDate().toInstant().isAfter(licenseReportRequest.getStartDate()) // + && dossier.getDate().toInstant().isBefore(licenseReportRequest.getEndDate()) // + && (dossier.getHardDeletedTime() == null || dossier.getHardDeletedTime() + .isAfter(OffsetDateTime.ofInstant(licenseReportRequest.getEndDate(), UTC_ZONE_ID)))).collect(Collectors.toSet()).size()) + .startDate(licenseReportRequest.getStartDate()) + .endDate(licenseReportRequest.getEndDate()) + .monthlyData(monthlyData) + .build(); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java index 1fb0cc684..51177eb27 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java @@ -66,11 +66,10 @@ public class LicenseReportTest extends AbstractPersistenceServerServiceTest { String requestId = "123"; request.setRequestId(requestId); - LicenseReport licenseReport = licenseReportClient.getLicenseReport(request, 0, 20); + LicenseReport licenseReport = licenseReportClient.getLicenseReport(request); assertThat(licenseReport.getNumberOfDossiers()).isEqualTo(dossiers.size()); assertThat(licenseReport.getNumberOfAnalyzedFiles()).isEqualTo(files.size()); - assertThat(licenseReport.getRequestId()).isEqualTo(requestId); assertThat(licenseReport.getStartDate()).isEqualTo(startDate); assertThat(licenseReport.getEndDate()).isEqualTo(endDate); } -- 2.47.2 From 2e7ca928eee53c393cd852b431f928465ab2dc6a Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Thu, 21 Sep 2023 17:31:40 +0300 Subject: [PATCH 2/3] RED-7326 - UT --- .../integration/tests/LicenseReportTest.java | 111 ++++++++++++++---- 1 file changed, 87 insertions(+), 24 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java index 51177eb27..117a274f9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/LicenseReportTest.java @@ -2,11 +2,10 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; import static org.assertj.core.api.Assertions.assertThat; +import java.time.Instant; import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.stream.Collectors; +import java.time.ZoneId; -import org.assertj.core.util.Lists; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,10 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTem import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.license.LicenseReport; import com.iqser.red.service.persistence.service.v1.api.model.license.LicenseReportRequest; @@ -35,43 +37,104 @@ public class LicenseReportTest extends AbstractPersistenceServerServiceTest { @Autowired private DossierTesterAndProvider dossierTesterAndProvider; + @Autowired + private FileRepository fileRepository; + + @Autowired + private DossierRepository dossierRepository; + @Test public void testLicenseReport() { var template = dossierTemplateTesterAndProvider.provideTestTemplate(); - var dossier1 = dossierTesterAndProvider.provideTestDossier(template, "Dossier1"); + var lastYearDossier = dossierTesterAndProvider.provideTestDossier(template, "lastYearDossier"); + addFileOnDate(lastYearDossier, "2022-01-01T10:00:00Z"); + addFileOnDate(lastYearDossier, "2022-01-02T10:00:00Z"); - var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, "test1.pdf"); - var file2 = fileTesterAndProvider.testAndProvideFile(dossier1, "test2.pdf"); + var januaryDossier = dossierTesterAndProvider.provideTestDossier(template, "januaryDossier"); + var januaryFile1 = addFileOnDate(januaryDossier, "2023-01-01T10:00:00Z"); + addFileOnDate(januaryDossier, "2023-01-02T10:00:00Z"); - var dossier2 = dossierTesterAndProvider.provideTestDossier(template, "Dossier2"); + var februaryDossier = dossierTesterAndProvider.provideTestDossier(template, "februaryDossier"); + addFileOnDate(februaryDossier, "2023-02-01T10:00:00Z"); + addFileOnDate(februaryDossier, "2023-02-02T10:00:00Z"); - var file3 = fileTesterAndProvider.testAndProvideFile(dossier2, "test3.pdf"); - var file4 = fileTesterAndProvider.testAndProvideFile(dossier2, "test4.pdf"); + var marchDossier = dossierTesterAndProvider.provideTestDossier(template, "marchDossier"); + addFileOnDate(marchDossier, "2023-03-01T10:00:00Z"); + addFileOnDate(marchDossier, "2023-03-02T10:00:00Z"); - var dossiers = Lists.newArrayList(dossier1, dossier2); - var files = Lists.newArrayList(file1, file2, file3, file4); + var aprilDossier = dossierTesterAndProvider.provideTestDossier(template, "aprilDossier"); + addFileOnDate(aprilDossier, "2023-04-01T10:00:00Z"); + addFileOnDate(aprilDossier, "2023-04-02T10:00:00Z"); - LicenseReportRequest request = new LicenseReportRequest(); - request.setDossierIds(dossiers.stream().map(Dossier::getId).collect(Collectors.toList())); + softDeleteFile(januaryFile1.getId(), "2023-02-01T10:00:00Z"); - var startDate = OffsetDateTime.now().minusHours(10).toInstant().truncatedTo(ChronoUnit.MILLIS); - request.setStartDate(startDate); + hardDeleteFile(januaryFile1.getId(), "2023-03-01T10:00:00Z"); - var endDate = OffsetDateTime.now().plusHours(10).toInstant().truncatedTo(ChronoUnit.MILLIS); - request.setEndDate(endDate); + archiveDossier(februaryDossier.getId(), "2023-05-01T10:00:00Z"); - String requestId = "123"; - request.setRequestId(requestId); + LicenseReport licenseReport = licenseReportClient.getLicenseReport(LicenseReportRequest.builder() + .startDate(Instant.parse("2023-01-01T10:00:00Z")) + .endDate(Instant.parse("2023-05-01T11:00:00Z")) + .build()); - LicenseReport licenseReport = licenseReportClient.getLicenseReport(request); + assertThat(licenseReport.getTotalFilesUploadedBytes()).isEqualTo(900L); + assertThat(licenseReport.getActiveFilesUploadedBytes()).isEqualTo(700L); + assertThat(licenseReport.getArchivedFilesUploadedBytes()).isEqualTo(200L); + assertThat(licenseReport.getTrashFilesUploadedBytes()).isEqualTo(0L); - assertThat(licenseReport.getNumberOfDossiers()).isEqualTo(dossiers.size()); - assertThat(licenseReport.getNumberOfAnalyzedFiles()).isEqualTo(files.size()); - assertThat(licenseReport.getStartDate()).isEqualTo(startDate); - assertThat(licenseReport.getEndDate()).isEqualTo(endDate); + assertThat(licenseReport.getMonthlyData().size()).isEqualTo(5); + assertThat(licenseReport.getMonthlyData().get(1).getTrashFilesUploadedBytes()).isEqualTo(100L); + assertThat(licenseReport.getMonthlyData().get(2).getTrashFilesUploadedBytes()).isEqualTo(0L); + } + + + private FileModel addFileOnDate(Dossier dossier, String date) { + + var file = fileTesterAndProvider.testAndProvideFile(dossier, date + ".pdf"); + setAdded(file.getId(), date); + return file; + } + + + private void setAdded(String fileId, String date) { + + fileRepository.findById(fileId).ifPresent((file) -> { + + file.setAdded(OffsetDateTime.ofInstant(Instant.parse(date), ZoneId.of("Z"))); + file.setFileSize(100L); + fileRepository.saveAndFlush(file); + + }); + } + + + private void archiveDossier(String dossierId, String date) { + + dossierRepository.findById(dossierId).ifPresent((dossier) -> { + dossier.setArchivedTime(OffsetDateTime.ofInstant(Instant.parse(date), ZoneId.of("Z"))); + dossierRepository.saveAndFlush(dossier); + }); + } + + + private void softDeleteFile(String fileId, String date) { + + fileRepository.findById(fileId).ifPresent((file) -> { + file.setDeleted(OffsetDateTime.ofInstant(Instant.parse(date), ZoneId.of("Z"))); + fileRepository.saveAndFlush(file); + }); + } + + + private void hardDeleteFile(String fileId, String date) { + + fileRepository.findById(fileId).ifPresent((file) -> { + file.setHardDeletedTime(OffsetDateTime.ofInstant(Instant.parse(date), ZoneId.of("Z"))); + fileRepository.saveAndFlush(file); + }); } } -- 2.47.2 From 6510e0577401cf590abf57383a33822ca087a49a Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Thu, 21 Sep 2023 17:42:05 +0300 Subject: [PATCH 3/3] RED-7326 - Fix PMD --- .../peristence/v1/server/service/FileStatusService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 08730e918..34a13cefa 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -40,7 +40,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; @@ -108,7 +107,7 @@ public class FileStatusService { public List getStatusesAddedBefore(OffsetDateTime end) { var fileEntities = fileStatusPersistenceService.getStatusesAddedBefore(end); - return MagicConverter.convert(fileEntities, FileModel.class, new FileModelMapper()); + return convert(fileEntities, FileModel.class, new FileModelMapper()); } -- 2.47.2