diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index bb57d29b8..7cc882425 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -228,6 +228,10 @@ public class FileStatusPersistenceService { return fileRepository.findByDossierId(dossierId); } + public List getStatusesForDossiersAndTimePeriod(Set dossierIds, OffsetDateTime start, OffsetDateTime end) { + + return fileRepository.findByDossierIdInAndAddedBetween(dossierIds, start, end); + } public List getActiveFiles(String dossierId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index 3d574dc3e..b870343fa 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -1,17 +1,19 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Set; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; - -import java.time.OffsetDateTime; -import java.util.List; public interface FileRepository extends JpaRepository { @@ -20,6 +22,8 @@ public interface FileRepository extends JpaRepository { List findByDossierId(String dossierId); + List findByDossierIdInAndAddedBetween(Set dossierIds, OffsetDateTime start, OffsetDateTime end); + @Modifying @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," + diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index b08a816a7..69d08bb73 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -82,10 +82,7 @@ public class FileStatusService { var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries()); var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); - return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList) - .stream() - .filter(FileModel::isAnalysisRequired) - .collect(Collectors.toList()); + return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList()); } @@ -107,6 +104,14 @@ public class FileStatusService { } + public List getStatusesForDossiersAndTimePeriod(Set dossierIds, OffsetDateTime start, OffsetDateTime end) { + + var fileEntities = fileStatusPersistenceService.getStatusesForDossiersAndTimePeriod(dossierIds, start, end); + var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); + return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); + } + + public List getSoftDeletedForDossierList(List dossierIds) { var fileEntities = fileStatusPersistenceService.getSoftDeletedFiles(dossierIds); @@ -124,8 +129,7 @@ public class FileStatusService { } - public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, - long fileSize) { + public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) { fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize); addToAnalysisQueue(dossierId, fileId, false, Set.of(), false); @@ -175,13 +179,22 @@ public class FileStatusService { public void setStatusSuccessful(String fileId, AnalyzeResult analyzeResult) { - fileStatusPersistenceService.updateProcessingStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), analyzeResult.getAnalysisNumber()); + fileStatusPersistenceService.updateProcessingStatus( + fileId, + analyzeResult.getNumberOfPages(), + analyzeResult.getDictionaryVersion(), + analyzeResult.getRulesVersion(), + analyzeResult.getLegalBasisVersion(), + analyzeResult.getDuration(), + analyzeResult.getDossierDictionaryVersion(), + analyzeResult.getAnalysisVersion(), + analyzeResult.getAnalysisNumber()); } @Transactional - public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse, - boolean triggeredManually, boolean manualRedactionReanalyse) { + public void setStatusReprocess( + String dossierId, String fileId, boolean priority, Set sectionsToReanalyse, boolean triggeredManually, boolean manualRedactionReanalyse) { log.info("Reprocessing file: {} from dossier {}", fileId, dossierId); @@ -215,8 +228,7 @@ public class FileStatusService { @Transactional - public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, - boolean requiresStructureAnalysis) { + public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -238,11 +250,7 @@ public class FileStatusService { @SneakyThrows public void addToPreprocessingQueue(String dossierId, String fileId, String filename) { - var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .fileName(filename) - .build(); + var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder().dossierId(dossierId).fileId(fileId).fileName(filename).build(); setStatusPreProcessingQueued(fileId); @@ -391,15 +399,18 @@ public class FileStatusService { setStatusImageAnalyzing(fileId); try { - rabbitTemplate.convertAndSend(MessagingConfiguration.IMAGE_SERVICE_QUEUE, objectMapper.writeValueAsString(ImageServiceRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .targetFileExtension(ImageServiceRequest.TARGET_FILE_EXTENSION) - .responseFileExtension(ImageServiceRequest.RESPONSE_FILE_EXTENSION) - .build()), message -> { - message.getMessageProperties().setPriority(1); - return message; - }); + rabbitTemplate.convertAndSend( + MessagingConfiguration.IMAGE_SERVICE_QUEUE, + objectMapper.writeValueAsString(ImageServiceRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .targetFileExtension(ImageServiceRequest.TARGET_FILE_EXTENSION) + .responseFileExtension(ImageServiceRequest.RESPONSE_FILE_EXTENSION) + .build()), + message -> { + message.getMessageProperties().setPriority(1); + return message; + }); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -409,16 +420,19 @@ public class FileStatusService { public void addToPdf2ImageQueue(String dossierId, String fileId, Set pages) { try { - rabbitTemplate.convertAndSend(MessagingConfiguration.PDF_2_IMAGE_QUEUE, objectMapper.writeValueAsString(Pdf2ImageServiceRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .targetFileExtension(Pdf2ImageServiceRequest.TARGET_FILE_EXTENSION) - .pages(pages) - .operation(Pdf2ImageServiceRequest.OPERATION_CONVERSION) - .build()), message -> { - message.getMessageProperties().setPriority(1); - return message; - }); + rabbitTemplate.convertAndSend( + MessagingConfiguration.PDF_2_IMAGE_QUEUE, + objectMapper.writeValueAsString(Pdf2ImageServiceRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .targetFileExtension(Pdf2ImageServiceRequest.TARGET_FILE_EXTENSION) + .pages(pages) + .operation(Pdf2ImageServiceRequest.OPERATION_CONVERSION) + .build()), + message -> { + message.getMessageProperties().setPriority(1); + return message; + }); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -428,15 +442,18 @@ public class FileStatusService { public void addToCvAnalysisRequestQueue(String dossierId, String fileId, Set pages) { try { - rabbitTemplate.convertAndSend(MessagingConfiguration.CV_ANALYSIS_QUEUE, objectMapper.writeValueAsString(CvAnalysisServiceRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .pages(pages) - .operation(CvAnalysisServiceRequest.OPERATION_TABLE_PARSING) - .build()), message -> { - message.getMessageProperties().setPriority(1); - return message; - }); + rabbitTemplate.convertAndSend( + MessagingConfiguration.CV_ANALYSIS_QUEUE, + objectMapper.writeValueAsString(CvAnalysisServiceRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .pages(pages) + .operation(CvAnalysisServiceRequest.OPERATION_TABLE_PARSING) + .build()), + message -> { + message.getMessageProperties().setPriority(1); + return message; + }); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -450,8 +467,7 @@ public class FileStatusService { @Transactional - public void overwriteFile(String dossierId, String fileId, String uploader, String filename, - boolean keepManualRedactions) { + public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean keepManualRedactions) { fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN); fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG); @@ -488,8 +504,7 @@ public class FileStatusService { // wipe force redactions var forceRedactions = forceRedactionPersistenceService.findForceRedactions(fileId, false); - forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); // wipe add manual redactions var addRedactions = addRedactionPersistenceService.findAddRedactions(fileId, false); @@ -497,23 +512,19 @@ public class FileStatusService { // wipe removeRedactions var removeRedactions = removeRedactionPersistenceService.findRemoveRedactions(fileId, false); - removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); // wipe image recat var imageRecategorizations = imageRecategorizationPersistenceService.findRecategorizations(fileId, false); - imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now));// wipe image recat + imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat // wipe resize redactions var resizeRedactions = resizeRedactionPersistenceService.findResizeRedactions(fileId, false); - resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); // wipe legal basis changes var legalBasisChanges = legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false); - legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId() - .getAnnotationId(), now)); + legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now)); fileStatusPersistenceService.updateHasComments(fileId, false); } @@ -547,8 +558,7 @@ public class FileStatusService { @Transactional - protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, - Set sectionsToReanalyse, boolean manualRedactionReanalyse) { + protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set sectionsToReanalyse, boolean manualRedactionReanalyse) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); var fileEntity = fileStatusPersistenceService.getStatus(fileId); @@ -619,23 +629,25 @@ public class FileStatusService { setStatusNerAnalyzing(fileId); try { - rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder() - .dossierId(dossierId) - .fileId(fileId) - .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION) - .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION) - .build()), message -> { - message.getMessageProperties().setPriority(1); - return message; - }); + rabbitTemplate.convertAndSend( + MessagingConfiguration.NER_SERVICE_QUEUE, + objectMapper.writeValueAsString(NerServiceRequest.builder() + .dossierId(dossierId) + .fileId(fileId) + .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION) + .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION) + .build()), + message -> { + message.getMessageProperties().setPriority(1); + return message; + }); } catch (JsonProcessingException e) { throw new RuntimeException(e); } } - private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, - ProcessingStatus processingStatus, FileModel fileModel) { + private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) { if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) { return MessageType.STRUCTURE_ANALYSE; @@ -644,8 +656,7 @@ public class FileStatusService { return MessageType.ANALYSE; } if (reanalyse) { - if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed() - .isBefore(fileModel.getLastFileAttributeChange())) { + if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) { return MessageType.ANALYSE; } return MessageType.REANALYSE; @@ -667,8 +678,7 @@ public class FileStatusService { } - private List convertAttributes( - List fileAttributes, String dossierTemplateId) { + private List convertAttributes(List fileAttributes, String dossierTemplateId) { List fileAttributeList = new ArrayList<>(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java index 1c0b59af3..7a61cd516 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java @@ -4,11 +4,16 @@ import static java.util.stream.Collectors.toList; import java.time.Duration; import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -32,6 +37,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class LicenseReportService { + private static final ZoneId UTC_ZONE_ID = ZoneId.of("Z"); private final FileStatusService fileStatusService; private final DossierService dossierService; @@ -80,6 +86,7 @@ public class LicenseReportService { } + private CachedReportData loadReport(LicenseReportRequest licenseReportRequest) { log.debug("No licence-report found in cache, generating new report"); @@ -88,49 +95,42 @@ public class LicenseReportService { start = Instant.now(); } - List dossierIds = licenseReportRequest.getDossierIds(); + final Set dossierIds; if (CollectionUtils.isEmpty(licenseReportRequest.getDossierIds())) { - dossierIds = dossierService.getAllDossiers().stream().map(DossierEntity::getId).collect(toList()); + dossierIds = dossierService.getAllDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet()); + } else { + dossierIds = new HashSet<>(licenseReportRequest.getDossierIds()); } AtomicInteger totalPagesAnalyzed = new AtomicInteger(); AtomicInteger totalPagesOcr = new AtomicInteger(); AtomicInteger totalNumberOfAnalyses = new AtomicInteger(); - List data = dossierIds.stream() - .flatMap(dossierId -> fileStatusService.getDossierStatus(dossierId) - .stream() - .filter(fileStatus -> fileStatus.getAdded().toInstant().isAfter(licenseReportRequest.getStartDate()) - && fileStatus.getAdded().toInstant().isBefore(licenseReportRequest.getEndDate())) - .map(fileStatus -> { - ReportData reportData = new ReportData(); - reportData.setDossier(dossierId); - reportData.setFileName(fileStatus.getFilename()); - reportData.setAddedDate(fileStatus.getAdded().toInstant()); - reportData.setLastUpdatedDate(fileStatus.getLastUpdated() == null ? null : fileStatus.getLastUpdated().toInstant()); - reportData.setDeletedDate(fileStatus.getDeleted() == null ? null : fileStatus.getDeleted().toInstant()); - reportData.setWorkflowStatus(fileStatus.getWorkflowStatus()); - reportData.setNumberOfAnalyzedPages(fileStatus.getNumberOfPages()); - reportData.setNumberOfOcrPages(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0); - reportData.setAnalysisCount(fileStatus.getNumberOfAnalyses()); - totalPagesAnalyzed.addAndGet(fileStatus.getNumberOfPages()); - totalPagesOcr.addAndGet(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0); - totalNumberOfAnalyses.addAndGet(fileStatus.getNumberOfAnalyses()); - return reportData; - })) - .sorted(Comparator.comparing(ReportData::getFileName)) - .collect(toList()); + List data = fileStatusService.getStatusesForDossiersAndTimePeriod( // + dossierIds, // + OffsetDateTime.ofInstant(licenseReportRequest.getStartDate(), UTC_ZONE_ID), // + OffsetDateTime.ofInstant(licenseReportRequest.getEndDate(), UTC_ZONE_ID)) // + .stream().map(fileStatus -> { + ReportData reportData = new ReportData(); + reportData.setDossier(fileStatus.getDossierId()); + reportData.setFileName(fileStatus.getFilename()); + reportData.setAddedDate(fileStatus.getAdded().toInstant()); + reportData.setLastUpdatedDate(fileStatus.getLastUpdated() == null ? null : fileStatus.getLastUpdated().toInstant()); + reportData.setDeletedDate(fileStatus.getDeleted() == null ? null : fileStatus.getDeleted().toInstant()); + reportData.setWorkflowStatus(fileStatus.getWorkflowStatus()); + reportData.setNumberOfAnalyzedPages(fileStatus.getNumberOfPages()); + reportData.setNumberOfOcrPages(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0); + reportData.setAnalysisCount(fileStatus.getNumberOfAnalyses()); + totalPagesAnalyzed.addAndGet(fileStatus.getNumberOfPages()); + totalPagesOcr.addAndGet(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0); + totalNumberOfAnalyses.addAndGet(fileStatus.getNumberOfAnalyses()); + return reportData; + }).sorted(Comparator.comparing(ReportData::getFileName)).collect(toList()); if (start != null) { log.info("loadReport took {} to process", Duration.between(start, Instant.now()).toString()); } - return new CachedReportData(data, - dossierIds.size(), - totalNumberOfAnalyses.get(), - totalPagesAnalyzed.get(), - totalPagesOcr.get(), - licenseReportRequest.getStartDate(), - licenseReportRequest.getEndDate()); + return new CachedReportData(data, dossierIds.size(), totalNumberOfAnalyses.get(), totalPagesAnalyzed.get(), totalPagesOcr.get(), licenseReportRequest.getStartDate(), licenseReportRequest.getEndDate()); } @@ -138,12 +138,13 @@ public class LicenseReportService { @Value class CachedReportData { - @NonNull - List data; + + @NonNull List data; int numberOfDossiers; int totalNumberOfAnalyses; int totalPagesAnalyzed; int totalOcrPages; Instant startDate; Instant endDate; + }