Merge branch 'RED-10286-bp' into 'release/2.589.x'

RED-10286: dont load all files at once, but only the ids

See merge request redactmanager/persistence-service!803

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/DossierExportService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
 
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -9,6 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
 import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
+import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
 import com.iqser.red.service.persistence.management.v1.processor.utils.FileSystemBackedArchiver;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
@@ -33,29 +35,30 @@ public class DossierExportService {
     FileStatusManagementService fileStatusManagementService;
     EntityTypeExportService entityTypeExportService;
     ObjectMapper mapper;
+    FileStatusService fileStatusService;
 
 
     @SneakyThrows
     @Observed(name = "FileExchangeExportService", contextualName = "export-dossier")
     public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) {
 
-        List<FileModel> files = fileStatusManagementService.getDossierStatus(dossier.getId());
+        List<String> fileIdsInDossier = fileStatusManagementService.getDossierStatusIds(dossier.getId(), false)
+                .stream()
+                .filter(fileId -> request.fileIds().isEmpty() || request.fileIds().contains(fileId))
+                .toList();
 
-        if (!request.dossierIds().contains(dossier.getId()) //
+        if (fileIdsInDossier.isEmpty()) {
-            && files.stream()
-                    .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) {
-            // dossier has no files in requested files and dossier not explicitly requested -> don't export it.
             return;
         }
 
         Path dossierFolder = folder.resolve(dossier.getId());
         archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier)));
 
-        for (FileModel fileEntity : files) {
+        for (String fileId : fileIdsInDossier) {
-            if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) {
+
-                continue;
+            FileModel file = fileStatusService.getStatus(fileId);
-            }
+
-            fileExportService.addFileToArchive(archiver, dossierFolder, request, fileEntity);
+            fileExportService.addFileToArchive(archiver, dossierFolder, request, file);
         }
 
         List<TypeEntity> types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
 
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage;
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
@@ -83,7 +86,7 @@ public class FileExchangeExportService {
 
 
     @Observed(name = "FileExchangeExportService", contextualName = "export-files")
-    public long createDownloadArchive(ExportDownloadMessage downloadJob) throws IOException {
+    public long createDownloadArchive(ExportDownloadMessage downloadJob) {
 
         FileExchangeExportRequest request = downloadJob.getFileExchangeExportRequest();
         DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(downloadJob.getDossierTemplateId());
@@ -92,13 +95,18 @@ public class FileExchangeExportService {
 
             dossierTemplateExportService.addDossierTemplateToArchive(archiver, FileExchangeNames.TEMPLATE_FOLDER, dossierTemplate);
 
-            for (Dossier dossierEntity : dossierManagementService.getAllDossiersForDossierTemplateId(dossierTemplate.getId(), true, false)) {
+            Iterable<String> dossierIds;
+            if (request.dossierIds().isEmpty()) {
+                dossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId(), true, false);
+            } else {
+                dossierIds = request.dossierIds();
+            }
 
-                if (!request.dossierIds().isEmpty() && !request.dossierIds().contains(dossierEntity.getId())) {
+            for (String dossierId : dossierIds) {
-                    continue;
-                }
 
-                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossierEntity);
+                Dossier dossier = dossierManagementService.getDossierById(dossierId, true, false);
+
+                dossierExportService.addDossierToArchive(archiver, Path.of(""), request, dossier);
             }
 
             storeZipFile(downloadJob.getStorageId(), archiver);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExportService.java

@@ -29,7 +29,6 @@ import lombok.extern.slf4j.Slf4j;
 public class FileExportService {
 
     ManualChangesExportService manualChangesExportService;
-    ComponentLogService componentLogService;
     ObjectMapper mapper;
     FileManagementStorageService storageService;
 
@@ -41,10 +40,12 @@ public class FileExportService {
         Path fileFolder = folder.resolve(file.getId());
 
         if (!request.excludeAnalysisLogs()) {
-            archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
+            if (storageService.entityLogExists(file.getDossierId(), file.getId())) {
-                                                                        buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
+                archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
-                                                                        mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
+                                                                            buildFileName(file, FileExchangeNames.ENTITY_LOG.getName()),
-            if (storageService.objectExists(file.getDossierId(), file.getId(), FileType.COMPONENT_LOG)) {
+                                                                            mapper.writeValueAsBytes(storageService.getEntityLog(file.getDossierId(), file.getId()))));
+            }
+            if (storageService.componentLogExists(file.getDossierId(), file.getId())) {
                 archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(fileFolder,
                                                                             buildFileName(file, FileExchangeNames.COMPONENT_LOG.getName()),
                                                                             mapper.writeValueAsBytes(storageService.getComponentLog(file.getDossierId(), file.getId()))));

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java

@@ -8,7 +8,7 @@ import org.springframework.stereotype.Service;
 
 import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
@@ -23,11 +23,12 @@ public class DossierIdFileIdRequestValidator {
     DossierManagementService dossierManagementService;
     FileStatusManagementService fileStatusManagementService;
 
+
     @Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request")
     public void validateRequestOrThrow404(String dossierTemplateId, Set<String> dossierIds, Set<String> fileIds) {
 
         if (!dossierIds.isEmpty()) {
-            Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
+            Set<String> availableDossierIds = new HashSet<>(dossierManagementService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, dossierIds));
             Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
             if (!nonAvailableDossiers.isEmpty()) {
                 throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId));
@@ -35,10 +36,20 @@ public class DossierIdFileIdRequestValidator {
         }
 
         if (!fileIds.isEmpty()) {
-            Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId)
+            Set<FileStatus> availableFiles = fileStatusManagementService.findAllByIds(fileIds)
                     .stream()
                     .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
-                    .map(FileModel::getId)
+                    .collect(Collectors.toSet());
+
+            Set<String> mentionedDossierIds = availableFiles.stream()
+                    .map(FileStatus::getDossierId)
+                    .collect(Collectors.toSet());
+
+            Set<String> availableDossierIds = new HashSet<>(dossierManagementService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, mentionedDossierIds));
+
+            Set<String> availableFileIds = availableFiles.stream()
+                    .filter(file -> availableDossierIds.contains(file.getDossierId()))
+                    .map(FileStatus::getFileId)
                     .collect(Collectors.toSet());
 
             Set<String> nonAvailableFiles = Sets.difference(fileIds, availableFileIds);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java

@@ -120,9 +120,15 @@ public class DossierManagementService {
     }
 
     @Transactional
-    public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
 
-        return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
+        return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
+    @Transactional
+    public List<String> findAllDossierIdsInDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
+
+        return dossierService.findAllDossierIdsInDossierTemplateId(dossierTemplateId, dossierIds);
 
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java

@@ -159,12 +159,17 @@ public class DossierService {
     }
 
 
-    public Set<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> findAllDossierIdsInDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
 
-        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId);
+        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId, dossierIds);
     }
 
 
+    public List<String> getAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
+
+        return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
     public Set<DossierChange> changesSince(OffsetDateTime since) {
 
         return dossierPersistenceService.hasChangesSince(since);
@@ -182,4 +187,6 @@ public class DossierService {
         dossierPersistenceService.unarchiveDossier(dossierId);
     }
 
+
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileManagementStorageService.java

@@ -170,6 +170,11 @@ public class FileManagementStorageService {
         return entityLogMongoService.entityLogDocumentExists(dossierId, fileId);
     }
 
+    public boolean componentLogExists(String dossierId, String fileId) {
+
+        return componentLogMongoService.componentLogDocumentExists(dossierId, fileId);
+    }
+
 
     public SectionGrid getSectionGrid(String dossierId, String fileId) {
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusManagementService.java

@@ -10,6 +10,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
@@ -23,7 +24,6 @@ public class FileStatusManagementService {
 
     private final FileStatusService fileStatusService;
     private final ExcludeFromAnalysisService excludeFromAnalysis;
-    private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
     private final IndexingService indexingService;
 
 
@@ -38,11 +38,13 @@ public class FileStatusManagementService {
         return fileStatusService.getAllFiles();
     }
 
+
     public List<FileModel> getAllDossierTemplateStatus(String dossierTemplateId) {
 
         return fileStatusService.getDossierTemplateStatus(dossierTemplateId);
     }
 
+
     public List<FileModel> getDossierStatus(String dossierId) {
 
         return fileStatusService.getDossierStatus(dossierId)
@@ -52,6 +54,11 @@ public class FileStatusManagementService {
     }
 
 
+    public List<String> getDossierStatusIds(String dossierId, boolean includeDeleted) {
+
+        return fileStatusService.getDossierStatusIds(dossierId, includeDeleted);
+    }
+
 
     public List<FileModel> getAllDossierStatus(String dossierId) {
 
@@ -194,4 +201,10 @@ public class FileStatusManagementService {
         fileStatusService.setExcludedPages(fileId, excludedPages);
     }
 
+
+    public List<FileStatus> findAllByIds(Set<String> fileIds) {
+
+        return fileStatusService.findAllByIds(fileIds);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileStatusService.java

@@ -61,6 +61,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.MessageType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.NerServiceRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
@@ -1066,4 +1067,16 @@ public class FileStatusService {
         addSearchTermOccurrencesAnalysisRequestToAnalysisQueue(fileStatus, bulkLocalRequest);
     }
 
+
+    public List<FileStatus> findAllByIds(Set<String> fileIds) {
+
+        return fileStatusPersistenceService.findAllByIds(fileIds).stream().map(entity -> MagicConverter.convert(entity, FileStatus.class)).collect(Collectors.toList());
+    }
+
+
+    public List<String> getDossierStatusIds(String dossierId, boolean includeDeleted) {
+
+        return fileStatusPersistenceService.findAllByDossierId(dossierId, includeDeleted);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java

@@ -4,6 +4,7 @@ import static com.iqser.red.service.persistence.management.v1.processor.exceptio
 
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -195,9 +196,9 @@ public class DossierPersistenceService {
     }
 
 
-    public Set<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId) {
+    public List<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId, Set<String> dossierIds) {
 
-        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId);
+        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId, dossierIds);
     }
 
 
@@ -282,4 +283,10 @@ public class DossierPersistenceService {
                 .orElseThrow(() -> new DossierNotFoundException(DOSSIER_NOT_FOUND_MESSAGE));
     }
 
+
+    public List<String> findAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) {
+
+        return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted);
+    }
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java

@@ -703,5 +703,15 @@ public class FileStatusPersistenceService {
     }
 
 
+    public List<FileEntity> findAllByIds(Set<String> fileIds) {
+
+        return fileRepository.findAllById(fileIds);
+    }
+
+
+    public List<String> findAllByDossierId(String dossierId, boolean includeDeleted) {
+
+        return fileRepository.findAllByDossierId(dossierId,  includeDeleted);
+    }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java

@@ -86,8 +86,18 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
     List<DossierEntity> findByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
 
 
-    @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId")
+    @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.id in :dossierIds")
-    Set<String> findIdsByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
+    List<String> findIdsByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId, @Param("dossierIds") Set<String> dossierIds);
+
+
+    @Query("select d.id from DossierEntity d "
+           + "where d.dossierTemplateId = :dossierTemplateId "
+           + "and (d.archivedTime is null or (d.archivedTime is not null and :includeArchived = true)) "
+           + "and (d.softDeletedTime is null or (d.softDeletedTime is not null and :includeDeleted = true)) "
+           + "and d.hardDeletedTime is null")
+    List<String> findIdsByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId,
+                                            @Param("includeArchived") boolean includeArchived,
+                                            @Param("includeDeleted") boolean includeDeleted);
 
 
     @Modifying
@@ -115,4 +125,8 @@ public interface DossierRepository extends JpaRepository<DossierEntity, String>
     @Query("SELECT d.dossierName FROM DossierEntity d WHERE d.id = :dossierId and d.hardDeletedTime IS NULL")
     Optional<String> findDossierNameById(@Param("dossierId") String dossierId);
 
+
+    @Query("select d from DossierEntity d where d.dossierTemplateId = :dossierTemplateId and d.archivedTime is null and d.softDeletedTime is null and d.hardDeletedTime is null")
+    List<DossierEntity> findActiveByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java

@@ -427,16 +427,19 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
     @Query("SELECT f FROM FileEntity f WHERE f.protobufMigrationDone = false")
     List<FileEntity> findByProtobufMigrationDoneFalse(Pageable pageable);
 
-    @Modifying
-    @Query("update FileEntity f set f.protobufMigrationDone = true "
-           + "where f.id in (:fileIds)")
-    void setProtobufMigrationDone(@Param("fileIds") List<String> fileIds);
 
+    @Modifying
+    @Query("update FileEntity f set f.protobufMigrationDone = true " + "where f.id in (:fileIds)")
+    void setProtobufMigrationDone(@Param("fileIds") List<String> fileIds);
 
 
     @Query("SELECT f FROM FileEntity f WHERE f.dossierId = :dossierId AND f.hardDeletedTime IS NULL AND f.deleted IS NULL")
     List<DossierStatsFileProjection> findDossierStatsProjectionFileProjectionByDossierId(@Param("dossierId") String dossierId);
 
+
+    @Query("SELECT f.id FROM FileEntity f WHERE f.dossierId = :dossierId AND f.hardDeletedTime IS NULL AND (f.deleted IS NULL or (f.deleted is not null and  :includeDeleted = true))")
+    List<String> findAllByDossierId(@Param("dossierId") String dossierId, @Param("includeDeleted") boolean includeDeleted);
+
 }
 
 

persistence-service-v1/persistence-service-shared-mongo-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/mongo/service/ComponentLogMongoService.java

@@ -128,4 +128,10 @@ public class ComponentLogMongoService {
         componentLogDocumentUpdateService.updateComponentLogEntryDocumentValues(mapper.getLogEntryId(dossierId, fileId, componentName), values, overridden);
     }
 
+
+    public boolean componentLogDocumentExists(String dossierId, String fileId) {
+
+        return componentLogDocumentRepository.existsById(mapper.getLogId(dossierId, fileId));
+    }
+
 }