RED-9255: implement file exchange

* add simplified text file to layoutparsing file export
* add validation for fileId/dossierId combinations

persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java

@@ -50,7 +50,6 @@ public class SupportController implements SupportResource {
     @Override
     public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
 
-        log.info("Scheduling reanalysis for all files in Dossier Tempalte {} with settings {}", dossierTemplateId, reanalysisSettings);
         reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings);
     }
 

persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java

@@ -56,6 +56,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionUpdateRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest;
 import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
@@ -216,12 +217,12 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
         Path mappingFile = saveToFile(file);
         String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename();
 
-        com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
+        ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
-                                                                                                                                                   nameToUse,
+                                                                           nameToUse,
-                                                                                                                                                   fileName,
+                                                                           fileName,
-                                                                                                                                                   delimiter,
+                                                                           delimiter,
-                                                                                                                                                   encoding,
+                                                                           encoding,
-                                                                                                                                                   mappingFile.toFile());
+                                                                           mappingFile.toFile());
 
         Files.deleteIfExists(mappingFile);
 

persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java

@@ -51,8 +51,23 @@ public interface SupportResource {
     String IMPORT = "/import";
 
 
-    @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
+    @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
-    @Operation(summary = "Reanalyze all files in dossier template", description = "None")
+    @Operation(summary = "Reanalyze all files in dossier template", description = """
+            ## Reanalyze Files Endpoint
+            
+            Use this endpoint to reanalyze all files in a specified Dossier Template. The reanalysis process can be tailored using various filtering options provided in the request body.
+            
+            ### Parameters
+            
+            - **DossierTemplateId**: Specifies the Dossier Template whose files need to be reanalyzed.
+            
+            ### Request Body Configuration Options
+            
+            - **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected for reanalysis.
+            - **fileIds**: List of file IDs to filter. If empty, all files are selected for reanalysis.
+            - **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated.
+            - **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null.
+            """)
     @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
     void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId,
                         @RequestBody ReanalysisSettings reAnalysisSettings);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java

@@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.management.v1.processor.dataexchange.Ex
 import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
+import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator;
 import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
 import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
@@ -61,6 +62,8 @@ public class FileExchangeExportService {
     ObjectMapper mapper;
     RabbitTemplate rabbitTemplate;
     EntityTypeExportService entityTypeExportService;
+    DossierIdFileIdRequestValidator requestValidator;
+
 
     @Observed(name = "FileExchangeExportService", contextualName = "prepare-export")
     public DownloadResponse prepareExportDownload(String dossierTemplateId, FileExchangeExportRequest request) {
@@ -69,7 +72,7 @@ public class FileExchangeExportService {
         DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId);
         String dossierTemplateName = dossierTemplate.getName();
 
-        validateRequest(dossierTemplate, request);
+        requestValidator.validateRequestOrThrow404(dossierTemplateId, request.dossierIds(), request.fileIds());
 
         String downloadFilename = dossierTemplateName + "_file-exchange.zip";
         String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_file-exchange");
@@ -87,42 +90,6 @@ public class FileExchangeExportService {
         return new DownloadResponse(storageId);
     }
 
-    @Observed(name = "FileExchangeExportService", contextualName = "validate-request")
-    private void validateRequest(DossierTemplateEntity dossierTemplate, FileExchangeExportRequest request) {
-
-        Set<String> dossierIds = new HashSet<>(request.dossierIds());
-        if (!request.dossierIds().isEmpty()) {
-            Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId());
-            Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
-            if (!nonAvailableDossiers.isEmpty()) {
-                throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s",
-                                                          String.join(", ", nonAvailableDossiers),
-                                                          dossierTemplate.getId()));
-            }
-        }
-
-        if (!request.fileIds().isEmpty()) {
-
-            Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplate.getId())
-                    .stream()
-                    .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
-                    .map(FileModel::getId)
-                    .collect(Collectors.toSet());
-
-            Set<String> nonAvailableFiles = Sets.difference(new HashSet<>(request.fileIds()), availableFileIds);
-
-            if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) {
-                throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplate.getId()));
-            }
-
-            if (!nonAvailableFiles.isEmpty()) {
-                throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s",
-                                                          String.join(", ", nonAvailableFiles),
-                                                          String.join(", ", dossierIds),
-                                                          dossierTemplate.getId()));
-            }
-        }
-    }
 
 
     private void addToExportDownloadQueue(ExportDownloadMessage downloadJob) {

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java

@@ -0,0 +1,59 @@
+package com.iqser.red.service.persistence.management.v1.processor.service;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.google.common.collect.Sets;
+import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DossierIdFileIdRequestValidator {
+
+    DossierManagementService dossierManagementService;
+    FileStatusManagementService fileStatusManagementService;
+
+    @Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request")
+    public void validateRequestOrThrow404(String dossierTemplateId, Set<String> dossierIds, Set<String> fileIds) {
+
+        if (!dossierIds.isEmpty()) {
+            Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
+            Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
+            if (!nonAvailableDossiers.isEmpty()) {
+                throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId));
+            }
+        }
+
+        if (!fileIds.isEmpty()) {
+            Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId)
+                    .stream()
+                    .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
+                    .map(FileModel::getId)
+                    .collect(Collectors.toSet());
+
+            Set<String> nonAvailableFiles = Sets.difference(fileIds, availableFileIds);
+
+            if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) {
+                throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplateId));
+            }
+
+            if (!nonAvailableFiles.isEmpty()) {
+                throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s",
+                                                          String.join(", ", nonAvailableFiles),
+                                                          String.join(", ", dossierIds),
+                                                          dossierTemplateId));
+            }
+        }
+    }
+
+}

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java

@@ -5,11 +5,13 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
 import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
 import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient;
+import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
 import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
 import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@@ -26,6 +28,7 @@ import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -41,6 +44,7 @@ public class ReanalysisService {
     private final IndexingService indexingService;
     private final PDFTronClient pDFTronRedactionClient;
     private final FileManagementStorageService fileManagementStorageService;
+    private final DossierIdFileIdRequestValidator requestValidator;
 
     private final Predicate<FileModel> validFilesFilter = fileStatus -> !fileStatus.isSoftOrHardDeleted() && !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED);
     private final Predicate<FileModel> errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR);
@@ -268,6 +272,7 @@ public class ReanalysisService {
 
     public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
 
+        requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds());
         fileStatusService.getDossierTemplateStatus(dossierTemplateId)
                 .stream()
                 .filter(file -> isInList(file, reanalysisSettings))

persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java

@@ -1,12 +1,13 @@
 package com.iqser.red.service.persistence.service.v1.api.shared.model;
 
 import java.util.List;
+import java.util.Set;
 
 import io.swagger.v3.oas.annotations.media.Schema;
 
 public record FileExchangeExportRequest(
-        @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") List<String> dossierIds,
+        @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") Set<String> dossierIds,
-        @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") List<String> fileIds,
+        @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") Set<String> fileIds,
         @Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles,
         @Schema(description = "If set to true, the MANUAL_REDACTIONS file will be excluded from the export.", defaultValue = "false") boolean excludeManualRedactions,
         @Schema(description = "If set to true, the ENTITY_LOG and COMPONENT_LOG (with its overrides) will be excluded from the export.", defaultValue = "false") boolean excludeAnalysisLogs,