RED-4928: Changed licence generation to have the database filter the data by the dossier-ids and dates from the request

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -228,6 +228,10 @@ public class FileStatusPersistenceService {
         return fileRepository.findByDossierId(dossierId);
     }
 
+    public List<FileEntity> getStatusesForDossiersAndTimePeriod(Set<String> dossierIds, OffsetDateTime start, OffsetDateTime end) {
+
+        return fileRepository.findByDossierIdInAndAddedBetween(dossierIds, start, end);
+    }
 
     public List<FileEntity> getActiveFiles(String dossierId) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -1,17 +1,19 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
 
+import java.time.OffsetDateTime;
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FilePageCountsProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileProcessingStatusProjection;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.projection.FileWorkflowStatusProjection;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Modifying;
-import org.springframework.data.jpa.repository.Query;
-
-import java.time.OffsetDateTime;
-import java.util.List;
 
 public interface FileRepository extends JpaRepository<FileEntity, String> {
 
@@ -20,6 +22,8 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
 
     List<FileEntity> findByDossierId(String dossierId);
 
+    List<FileEntity> findByDossierIdInAndAddedBetween(Set<String> dossierIds, OffsetDateTime start, OffsetDateTime end);
+
 
     @Modifying
     @Query("update FileEntity e set e.hasRedactions = :hasRedactions ," +

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java

@@ -82,10 +82,7 @@ public class FileStatusService {
 
         var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(fileManagementServiceSettings.getMaxErrorRetries());
         var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
-        return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList)
+        return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList).stream().filter(FileModel::isAnalysisRequired).collect(Collectors.toList());
-                .stream()
-                .filter(FileModel::isAnalysisRequired)
-                .collect(Collectors.toList());
     }
 
 
@@ -107,6 +104,14 @@ public class FileStatusService {
     }
 
 
+    public List<FileModel> getStatusesForDossiersAndTimePeriod(Set<String> dossierIds, OffsetDateTime start, OffsetDateTime end) {
+
+        var fileEntities = fileStatusPersistenceService.getStatusesForDossiersAndTimePeriod(dossierIds, start, end);
+        var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
+        return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
+    }
+
+
     public List<FileModel> getSoftDeletedForDossierList(List<String> dossierIds) {
 
         var fileEntities = fileStatusPersistenceService.getSoftDeletedFiles(dossierIds);
@@ -124,8 +129,7 @@ public class FileStatusService {
     }
 
 
-    public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights,
+    public void updateProcessingStatusPreprocessed(String dossierId, String fileId, boolean hasHighlights, long fileSize) {
-                                                   long fileSize) {
 
         fileStatusPersistenceService.updateProcessingStatusPreprocessed(fileId, hasHighlights, fileSize);
         addToAnalysisQueue(dossierId, fileId, false, Set.of(), false);
@@ -175,13 +179,22 @@ public class FileStatusService {
 
     public void setStatusSuccessful(String fileId, AnalyzeResult analyzeResult) {
 
-        fileStatusPersistenceService.updateProcessingStatus(fileId, analyzeResult.getNumberOfPages(), analyzeResult.getDictionaryVersion(), analyzeResult.getRulesVersion(), analyzeResult.getLegalBasisVersion(), analyzeResult.getDuration(), analyzeResult.getDossierDictionaryVersion(), analyzeResult.getAnalysisVersion(), analyzeResult.getAnalysisNumber());
+        fileStatusPersistenceService.updateProcessingStatus(
+                fileId,
+                analyzeResult.getNumberOfPages(),
+                analyzeResult.getDictionaryVersion(),
+                analyzeResult.getRulesVersion(),
+                analyzeResult.getLegalBasisVersion(),
+                analyzeResult.getDuration(),
+                analyzeResult.getDossierDictionaryVersion(),
+                analyzeResult.getAnalysisVersion(),
+                analyzeResult.getAnalysisNumber());
     }
 
 
     @Transactional
-    public void setStatusReprocess(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse,
+    public void setStatusReprocess(
-                                   boolean triggeredManually, boolean manualRedactionReanalyse) {
+            String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse, boolean triggeredManually, boolean manualRedactionReanalyse) {
 
         log.info("Reprocessing file: {} from dossier {}", fileId, dossierId);
 
@@ -215,8 +228,7 @@ public class FileStatusService {
 
 
     @Transactional
-    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority,
+    public void setStatusFullReprocess(String dossierId, String fileId, boolean priority, boolean requiresStructureAnalysis) {
-                                       boolean requiresStructureAnalysis) {
 
         FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
 
@@ -238,11 +250,7 @@ public class FileStatusService {
     @SneakyThrows
     public void addToPreprocessingQueue(String dossierId, String fileId, String filename) {
 
-        var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder()
+        var processUntouchedDocumentRequest = ProcessUntouchedDocumentRequest.builder().dossierId(dossierId).fileId(fileId).fileName(filename).build();
-                .dossierId(dossierId)
-                .fileId(fileId)
-                .fileName(filename)
-                .build();
 
         setStatusPreProcessingQueued(fileId);
 
@@ -391,15 +399,18 @@ public class FileStatusService {
 
         setStatusImageAnalyzing(fileId);
         try {
-            rabbitTemplate.convertAndSend(MessagingConfiguration.IMAGE_SERVICE_QUEUE, objectMapper.writeValueAsString(ImageServiceRequest.builder()
+            rabbitTemplate.convertAndSend(
-                    .dossierId(dossierId)
+                    MessagingConfiguration.IMAGE_SERVICE_QUEUE,
-                    .fileId(fileId)
+                    objectMapper.writeValueAsString(ImageServiceRequest.builder()
-                    .targetFileExtension(ImageServiceRequest.TARGET_FILE_EXTENSION)
+                            .dossierId(dossierId)
-                    .responseFileExtension(ImageServiceRequest.RESPONSE_FILE_EXTENSION)
+                            .fileId(fileId)
-                    .build()), message -> {
+                            .targetFileExtension(ImageServiceRequest.TARGET_FILE_EXTENSION)
-                message.getMessageProperties().setPriority(1);
+                            .responseFileExtension(ImageServiceRequest.RESPONSE_FILE_EXTENSION)
-                return message;
+                            .build()),
-            });
+                    message -> {
+                        message.getMessageProperties().setPriority(1);
+                        return message;
+                    });
         } catch (JsonProcessingException e) {
             throw new RuntimeException(e);
         }
@@ -409,16 +420,19 @@ public class FileStatusService {
     public void addToPdf2ImageQueue(String dossierId, String fileId, Set<Integer> pages) {
 
         try {
-            rabbitTemplate.convertAndSend(MessagingConfiguration.PDF_2_IMAGE_QUEUE, objectMapper.writeValueAsString(Pdf2ImageServiceRequest.builder()
+            rabbitTemplate.convertAndSend(
-                    .dossierId(dossierId)
+                    MessagingConfiguration.PDF_2_IMAGE_QUEUE,
-                    .fileId(fileId)
+                    objectMapper.writeValueAsString(Pdf2ImageServiceRequest.builder()
-                    .targetFileExtension(Pdf2ImageServiceRequest.TARGET_FILE_EXTENSION)
+                            .dossierId(dossierId)
-                    .pages(pages)
+                            .fileId(fileId)
-                    .operation(Pdf2ImageServiceRequest.OPERATION_CONVERSION)
+                            .targetFileExtension(Pdf2ImageServiceRequest.TARGET_FILE_EXTENSION)
-                    .build()), message -> {
+                            .pages(pages)
-                message.getMessageProperties().setPriority(1);
+                            .operation(Pdf2ImageServiceRequest.OPERATION_CONVERSION)
-                return message;
+                            .build()),
-            });
+                    message -> {
+                        message.getMessageProperties().setPriority(1);
+                        return message;
+                    });
         } catch (JsonProcessingException e) {
             throw new RuntimeException(e);
         }
@@ -428,15 +442,18 @@ public class FileStatusService {
     public void addToCvAnalysisRequestQueue(String dossierId, String fileId, Set<Integer> pages) {
 
         try {
-            rabbitTemplate.convertAndSend(MessagingConfiguration.CV_ANALYSIS_QUEUE, objectMapper.writeValueAsString(CvAnalysisServiceRequest.builder()
+            rabbitTemplate.convertAndSend(
-                    .dossierId(dossierId)
+                    MessagingConfiguration.CV_ANALYSIS_QUEUE,
-                    .fileId(fileId)
+                    objectMapper.writeValueAsString(CvAnalysisServiceRequest.builder()
-                    .pages(pages)
+                            .dossierId(dossierId)
-                    .operation(CvAnalysisServiceRequest.OPERATION_TABLE_PARSING)
+                            .fileId(fileId)
-                    .build()), message -> {
+                            .pages(pages)
-                message.getMessageProperties().setPriority(1);
+                            .operation(CvAnalysisServiceRequest.OPERATION_TABLE_PARSING)
-                return message;
+                            .build()),
-            });
+                    message -> {
+                        message.getMessageProperties().setPriority(1);
+                        return message;
+                    });
         } catch (JsonProcessingException e) {
             throw new RuntimeException(e);
         }
@@ -450,8 +467,7 @@ public class FileStatusService {
 
 
     @Transactional
-    public void overwriteFile(String dossierId, String fileId, String uploader, String filename,
+    public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean keepManualRedactions) {
-                              boolean keepManualRedactions) {
 
         fileManagementStorageService.deleteObject(dossierId, fileId, FileType.ORIGIN);
         fileManagementStorageService.deleteObject(dossierId, fileId, FileType.REDACTION_LOG);
@@ -488,8 +504,7 @@ public class FileStatusService {
 
         // wipe force redactions
         var forceRedactions = forceRedactionPersistenceService.findForceRedactions(fileId, false);
-        forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId()
+        forceRedactions.forEach(f -> forceRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         // wipe add manual redactions
         var addRedactions = addRedactionPersistenceService.findAddRedactions(fileId, false);
@@ -497,23 +512,19 @@ public class FileStatusService {
 
         // wipe removeRedactions
         var removeRedactions = removeRedactionPersistenceService.findRemoveRedactions(fileId, false);
-        removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId()
+        removeRedactions.forEach(f -> removeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         // wipe image recat
         var imageRecategorizations = imageRecategorizationPersistenceService.findRecategorizations(fileId, false);
-        imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId()
+        imageRecategorizations.forEach(f -> imageRecategorizationPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));// wipe image recat
-                .getAnnotationId(), now));// wipe image recat
 
         // wipe resize redactions
         var resizeRedactions = resizeRedactionPersistenceService.findResizeRedactions(fileId, false);
-        resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId()
+        resizeRedactions.forEach(f -> resizeRedactionPersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         // wipe legal basis changes
         var legalBasisChanges = legalBasisChangePersistenceService.findLegalBasisChanges(fileId, false);
-        legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId()
+        legalBasisChanges.forEach(f -> legalBasisChangePersistenceService.softDelete(fileId, f.getId().getAnnotationId(), now));
-                .getAnnotationId(), now));
 
         fileStatusPersistenceService.updateHasComments(fileId, false);
     }
@@ -547,8 +558,7 @@ public class FileStatusService {
 
 
     @Transactional
-    protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority,
+    protected void addToAnalysisQueue(String dossierId, String fileId, boolean priority, Set<Integer> sectionsToReanalyse, boolean manualRedactionReanalyse) {
-                                      Set<Integer> sectionsToReanalyse, boolean manualRedactionReanalyse) {
 
         var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
         var fileEntity = fileStatusPersistenceService.getStatus(fileId);
@@ -619,23 +629,25 @@ public class FileStatusService {
 
         setStatusNerAnalyzing(fileId);
         try {
-            rabbitTemplate.convertAndSend(MessagingConfiguration.NER_SERVICE_QUEUE, objectMapper.writeValueAsString(NerServiceRequest.builder()
+            rabbitTemplate.convertAndSend(
-                    .dossierId(dossierId)
+                    MessagingConfiguration.NER_SERVICE_QUEUE,
-                    .fileId(fileId)
+                    objectMapper.writeValueAsString(NerServiceRequest.builder()
-                    .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION)
+                            .dossierId(dossierId)
-                    .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION)
+                            .fileId(fileId)
-                    .build()), message -> {
+                            .targetFileExtension(NerServiceRequest.TARGET_FILE_EXTENSION)
-                message.getMessageProperties().setPriority(1);
+                            .responseFileExtension(NerServiceRequest.RESPONSE_FILE_EXTENSION)
-                return message;
+                            .build()),
-            });
+                    message -> {
+                        message.getMessageProperties().setPriority(1);
+                        return message;
+                    });
         } catch (JsonProcessingException e) {
             throw new RuntimeException(e);
         }
     }
 
 
-    private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
+    private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus, FileModel fileModel) {
-                                             ProcessingStatus processingStatus, FileModel fileModel) {
 
         if (!fileManagementStorageService.objectExists(dossierId, fileId, FileType.TEXT)) {
             return MessageType.STRUCTURE_ANALYSE;
@@ -644,8 +656,7 @@ public class FileStatusService {
             return MessageType.ANALYSE;
         }
         if (reanalyse) {
-            if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed()
+            if (fileModel.getLastFileAttributeChange() != null && fileModel.getLastProcessed().isBefore(fileModel.getLastFileAttributeChange())) {
-                    .isBefore(fileModel.getLastFileAttributeChange())) {
                 return MessageType.ANALYSE;
             }
             return MessageType.REANALYSE;
@@ -667,8 +678,7 @@ public class FileStatusService {
     }
 
 
-    private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(
+    private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
-            List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
 
         List<com.iqser.red.service.redaction.v1.model.FileAttribute> fileAttributeList = new ArrayList<>();
 

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/LicenseReportService.java

@@ -4,11 +4,16 @@ import static java.util.stream.Collectors.toList;
 
 import java.time.Duration;
 import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -32,6 +37,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 public class LicenseReportService {
 
+    private static final ZoneId UTC_ZONE_ID = ZoneId.of("Z");
     private final FileStatusService fileStatusService;
     private final DossierService dossierService;
 
@@ -80,6 +86,7 @@ public class LicenseReportService {
 
     }
 
+
     private CachedReportData loadReport(LicenseReportRequest licenseReportRequest) {
 
         log.debug("No licence-report found in cache, generating new report");
@@ -88,49 +95,42 @@ public class LicenseReportService {
             start = Instant.now();
         }
 
-        List<String> dossierIds = licenseReportRequest.getDossierIds();
+        final Set<String> dossierIds;
         if (CollectionUtils.isEmpty(licenseReportRequest.getDossierIds())) {
-            dossierIds = dossierService.getAllDossiers().stream().map(DossierEntity::getId).collect(toList());
+            dossierIds = dossierService.getAllDossiers().stream().map(DossierEntity::getId).collect(Collectors.toSet());
+        } else {
+            dossierIds = new HashSet<>(licenseReportRequest.getDossierIds());
         }
 
         AtomicInteger totalPagesAnalyzed = new AtomicInteger();
         AtomicInteger totalPagesOcr = new AtomicInteger();
         AtomicInteger totalNumberOfAnalyses = new AtomicInteger();
-        List<ReportData> data = dossierIds.stream()
+        List<ReportData> data = fileStatusService.getStatusesForDossiersAndTimePeriod( //
-                .flatMap(dossierId -> fileStatusService.getDossierStatus(dossierId)
+                        dossierIds, //
-                        .stream()
+                        OffsetDateTime.ofInstant(licenseReportRequest.getStartDate(), UTC_ZONE_ID), //
-                        .filter(fileStatus -> fileStatus.getAdded().toInstant().isAfter(licenseReportRequest.getStartDate())
+                        OffsetDateTime.ofInstant(licenseReportRequest.getEndDate(), UTC_ZONE_ID)) //
-                                && fileStatus.getAdded().toInstant().isBefore(licenseReportRequest.getEndDate()))
+                .stream().map(fileStatus -> {
-                        .map(fileStatus -> {
+                    ReportData reportData = new ReportData();
-                            ReportData reportData = new ReportData();
+                    reportData.setDossier(fileStatus.getDossierId());
-                            reportData.setDossier(dossierId);
+                    reportData.setFileName(fileStatus.getFilename());
-                            reportData.setFileName(fileStatus.getFilename());
+                    reportData.setAddedDate(fileStatus.getAdded().toInstant());
-                            reportData.setAddedDate(fileStatus.getAdded().toInstant());
+                    reportData.setLastUpdatedDate(fileStatus.getLastUpdated() == null ? null : fileStatus.getLastUpdated().toInstant());
-                            reportData.setLastUpdatedDate(fileStatus.getLastUpdated() == null ? null : fileStatus.getLastUpdated().toInstant());
+                    reportData.setDeletedDate(fileStatus.getDeleted() == null ? null : fileStatus.getDeleted().toInstant());
-                            reportData.setDeletedDate(fileStatus.getDeleted() == null ? null : fileStatus.getDeleted().toInstant());
+                    reportData.setWorkflowStatus(fileStatus.getWorkflowStatus());
-                            reportData.setWorkflowStatus(fileStatus.getWorkflowStatus());
+                    reportData.setNumberOfAnalyzedPages(fileStatus.getNumberOfPages());
-                            reportData.setNumberOfAnalyzedPages(fileStatus.getNumberOfPages());
+                    reportData.setNumberOfOcrPages(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0);
-                            reportData.setNumberOfOcrPages(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0);
+                    reportData.setAnalysisCount(fileStatus.getNumberOfAnalyses());
-                            reportData.setAnalysisCount(fileStatus.getNumberOfAnalyses());
+                    totalPagesAnalyzed.addAndGet(fileStatus.getNumberOfPages());
-                            totalPagesAnalyzed.addAndGet(fileStatus.getNumberOfPages());
+                    totalPagesOcr.addAndGet(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0);
-                            totalPagesOcr.addAndGet(fileStatus.getLastOCRTime() != null ? fileStatus.getNumberOfPages() : 0);
+                    totalNumberOfAnalyses.addAndGet(fileStatus.getNumberOfAnalyses());
-                            totalNumberOfAnalyses.addAndGet(fileStatus.getNumberOfAnalyses());
+                    return reportData;
-                            return reportData;
+                }).sorted(Comparator.comparing(ReportData::getFileName)).collect(toList());
-                        }))
-                .sorted(Comparator.comparing(ReportData::getFileName))
-                .collect(toList());
 
         if (start != null) {
             log.info("loadReport took {} to process", Duration.between(start, Instant.now()).toString());
         }
 
-        return new CachedReportData(data,
+        return new CachedReportData(data, dossierIds.size(), totalNumberOfAnalyses.get(), totalPagesAnalyzed.get(), totalPagesOcr.get(), licenseReportRequest.getStartDate(), licenseReportRequest.getEndDate());
-                dossierIds.size(),
-                totalNumberOfAnalyses.get(),
-                totalPagesAnalyzed.get(),
-                totalPagesOcr.get(),
-                licenseReportRequest.getStartDate(),
-                licenseReportRequest.getEndDate());
 
     }
 
@@ -138,12 +138,13 @@ public class LicenseReportService {
 
 @Value
 class CachedReportData {
-    @NonNull
+
-    List<ReportData> data;
+    @NonNull List<ReportData> data;
     int numberOfDossiers;
     int totalNumberOfAnalyses;
     int totalPagesAnalyzed;
     int totalOcrPages;
     Instant startDate;
     Instant endDate;
+
 }