From 27d2208a72736da67f83498c3b13c71775a6e43d Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 17 Jun 2024 16:21:56 +0200 Subject: [PATCH 01/13] RED-9255: give reanalysis endpoint a request model --- .../api/impl/controller/SupportController.java | 6 ++++-- .../v1/api/external/resource/SupportResource.java | 3 ++- .../src/main/resources/api/documine.yaml | 2 +- .../v1/processor/service/ReanalysisService.java | 15 +++++++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index 80bbdc251..bf6dd6756 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -23,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper; import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService; +import com.iqser.red.service.persistence.service.v1.api.external.model.ReAnalysisSettings; import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; @@ -47,9 +48,10 @@ public class SupportController implements SupportResource { @Override - public void reanalyzeFiles(String dossierTemplateId, boolean repeatStructureAnalysis) { + public void reanalyzeFiles(String dossierTemplateId, ReAnalysisSettings reAnalysisSettings) { - reanalysisService.reanalyzeTemplate(dossierTemplateId, repeatStructureAnalysis); + log.info("Scheduling reanalysis for all files in Dossier Tempalte {} with settings {}", dossierTemplateId, reAnalysisSettings); + reanalysisService.reanalyzeTemplate(dossierTemplateId, reAnalysisSettings); } diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index 1a8b4f61f..0a27536bb 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.service.v1.api.external.model.ReAnalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter; @@ -54,7 +55,7 @@ public interface SupportResource { @Operation(summary = "Reanalyze all files in dossier template", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")}) void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, - @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis); + @RequestBody ReAnalysisSettings reAnalysisSettings); @PostMapping(value = ERROR_REANALYSIS_REST_PATH) diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml index ab41b3627..765dd7e1f 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml @@ -594,7 +594,7 @@ paths: - Component Definitions summary: Create new component definitions description: | - Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case + Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case that can't be updated after the creation. The rank is used to determine the order in which the components are displayed. The component's rank will automatically be appended at the end based on the current number of components of this dossier template. parameters: diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 4e956f861..2edbef718 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -9,6 +9,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.external.model.ReAnalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -265,13 +266,23 @@ public class ReanalysisService { } - public void reanalyzeTemplate(String dossierTemplateId, boolean repeatStructureAnalysis) { + public void reanalyzeTemplate(String dossierTemplateId, ReAnalysisSettings reAnalysisSettings) { fileStatusService.getDossierTemplateStatus(dossierTemplateId) .stream() .filter(file -> !file.isSoftOrHardDeleted()) - .forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, repeatStructureAnalysis)); + .filter(file -> isInList(file, reAnalysisSettings)) + .filter(file -> file.getProcessingStatus().equals(ProcessingStatus.ERROR) || !reAnalysisSettings.onlyErrorFiles()) + .peek(file -> log.info("Reanalyzing file {}", file.getId())) + .forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reAnalysisSettings.repeatStructureAnalysis())); } + + private boolean isInList(FileModel file, ReAnalysisSettings reAnalysisSettings) { + + return (reAnalysisSettings.fileIds().isEmpty() || reAnalysisSettings.fileIds().contains(file.getId())) // + && (reAnalysisSettings.dossierIds().isEmpty() || reAnalysisSettings.dossierIds().contains(file.getDossierId())); + } + } From f77c08aac8097372e53c9e40f4c4557988f14f7c Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:13:18 +0200 Subject: [PATCH 02/13] RED-9255: implement file exchange * add simplified text file to layoutparsing file export * add validation for fileId/dossierId combinations --- .../external/resource/SupportResource.java | 4 +- .../dataexchange/FileExchangeNames.java | 1 + .../service/FileExchangeExportService.java | 66 ++++++++++++++++++- .../zipreaders/FileExchangeArchiveReader.java | 1 + .../service/DossierManagementService.java | 7 ++ .../v1/processor/service/DossierService.java | 17 +++-- .../DossierPersistenceService.java | 6 ++ .../repository/DossierRepository.java | 5 ++ .../model/FileExchangeExportRequest.java | 2 +- 9 files changed, 97 insertions(+), 12 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index 0a27536bb..8f89b665b 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -116,7 +116,7 @@ public interface SupportResource { - **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected. - **fileIds**: List of file IDs to filter. If empty, all files are selected. - - **excludeLayoutFiles**: Boolean. If true, excludes DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS and NER_ENTITIES files. + - **excludeLayoutFiles**: Boolean. If true, excludes DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files. - **excludeManualRedactions**: Boolean. If true, excludes MANUAL_REDACTIONS files. - **excludeAnalysisLogs**: Boolean. If true, excludes ENTITY_LOG and COMPONENT_LOG (with overrides) files. - **excludeFileAttributes**: Boolean. If true, excludes file attributes. @@ -128,7 +128,7 @@ public interface SupportResource { @ResponseStatus(value = HttpStatus.NO_CONTENT) @ResponseBody @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. If the request parameter dossierTemplateId is set, the files will be imported into the existing DossierTemplate.") + @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files.") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) void importFiles(@RequestPart(name = "file") MultipartFile file); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java index 8237ca925..1c3d948ea 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java @@ -24,6 +24,7 @@ public class FileExchangeNames { public static Definition PAGES = new Definition(FileType.DOCUMENT_PAGES); public static Definition TEXT = new Definition(FileType.DOCUMENT_TEXT); public static Definition POSITIONS = new Definition(FileType.DOCUMENT_POSITION); + public static Definition SIMPLIFIED_TEXT = new Definition(FileType.SIMPLIFIED_TEXT); public static Definition NER_ENTITIES = new Definition(FileType.NER_ENTITIES); public static Definition TABLES = new Definition(FileType.TABLES); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java index 390a95ae3..223a89ccd 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java @@ -3,17 +3,22 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; @@ -57,11 +62,15 @@ public class FileExchangeExportService { RabbitTemplate rabbitTemplate; EntityTypeExportService entityTypeExportService; - + @Observed(name = "FileExchangeExportService", contextualName = "prepare-export") public DownloadResponse prepareExportDownload(String dossierTemplateId, FileExchangeExportRequest request) { var mimeType = "application/zip"; - String dossierTemplateName = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getName(); + DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId); + String dossierTemplateName = dossierTemplate.getName(); + + validateRequest(dossierTemplate, request); + String downloadFilename = dossierTemplateName + "_file-exchange.zip"; String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_file-exchange"); @@ -78,6 +87,43 @@ public class FileExchangeExportService { return new DownloadResponse(storageId); } + @Observed(name = "FileExchangeExportService", contextualName = "validate-request") + private void validateRequest(DossierTemplateEntity dossierTemplate, FileExchangeExportRequest request) { + + Set dossierIds = new HashSet<>(request.dossierIds()); + if (!request.dossierIds().isEmpty()) { + Set availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId()); + Set nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds); + if (!nonAvailableDossiers.isEmpty()) { + throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", + String.join(", ", nonAvailableDossiers), + dossierTemplate.getId())); + } + } + + if (!request.fileIds().isEmpty()) { + + Set availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplate.getId()) + .stream() + .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId())) + .map(FileModel::getId) + .collect(Collectors.toSet()); + + Set nonAvailableFiles = Sets.difference(new HashSet<>(request.fileIds()), availableFileIds); + + if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) { + throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplate.getId())); + } + + if (!nonAvailableFiles.isEmpty()) { + throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s", + String.join(", ", nonAvailableFiles), + String.join(", ", dossierIds), + dossierTemplate.getId())); + } + } + } + private void addToExportDownloadQueue(ExportDownloadMessage downloadJob) { @@ -118,14 +164,26 @@ public class FileExchangeExportService { @Observed(name = "FileExchangeExportService", contextualName = "export-dossier") public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) { + List files = fileStatusManagementService.getDossierStatus(dossier.getId()); + + if (!request.fileIds().isEmpty() // + && !request.dossierIds().isEmpty() // + && files.stream() + .noneMatch(fileModel -> request.fileIds().contains(fileModel.getId()))) { + // no files of dossier in requested files and dossier not explicitly requested -> don't export it. + return; + } + Path dossierFolder = folder.resolve(dossier.getId()); archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier))); - for (FileModel fileEntity : fileStatusManagementService.getDossierStatus(dossier.getId())) { + + for (FileModel fileEntity : files) { if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) { continue; } addFileToArchive(archiver, dossierFolder, request, fileEntity); } + List types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false); String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString(); @@ -164,6 +222,7 @@ public class FileExchangeExportService { addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS); addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES); addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES); + addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT); } addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE); @@ -220,6 +279,7 @@ public class FileExchangeExportService { archiver.addEntry(archiveModel); } + @Observed(name = "FileExchangeExportService", contextualName = "store-archive") private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java index a1a1db76c..d3ef43887 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java @@ -97,6 +97,7 @@ public class FileExchangeArchiveReader { FileExchangeNames.PAGES, FileExchangeNames.TEXT, FileExchangeNames.POSITIONS, + FileExchangeNames.SIMPLIFIED_TEXT, FileExchangeNames.NER_ENTITIES, FileExchangeNames.TABLES, FileExchangeNames.IMAGES, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java index 130317c51..7ef356635 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java @@ -119,6 +119,13 @@ public class DossierManagementService { } + @Transactional + public Set getAllDossierIdsForDossierTemplateId(String dossierTemplateId) { + + return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId); + + } + public DossierInformation getDossierInformation(List filteredDossierIds) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java index 0b69d3073..b7ddd096b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java @@ -1,5 +1,11 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Set; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -17,12 +23,6 @@ import jakarta.validation.ConstraintViolationException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.util.List; -import java.util.Set; - /** * Provides the internal interface between dossier request and the actual persistence. */ @@ -143,6 +143,11 @@ public class DossierService { return dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId); } + public Set getAllDossierIdsForDossierTemplateId(String dossierTemplateId) { + + return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId); + } + public Set changesSince(OffsetDateTime since) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index 3046c748c..a4e10a712 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -195,6 +195,12 @@ public class DossierPersistenceService { } + public Set findAllDossierIdsForDossierTemplateId(String dossierTemplateId) { + + return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId); + } + + @Transactional public void hardDelete(String dossierId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 667d5e7c4..247d3c8bc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import java.time.OffsetDateTime; import java.util.List; +import java.util.Set; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; @@ -84,6 +85,10 @@ public interface DossierRepository extends JpaRepository List findByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId); + @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId") + Set findIdsByDossierTemplateId(String dossierTemplateId); + + @Modifying @Query("update DossierEntity d set d.watermarkId = null where d.watermarkId = :watermarkId") int countDeleteWatermark(@Param("watermarkId") long watermarkId); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java index c5af4f4bc..15783d39e 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java @@ -7,7 +7,7 @@ import io.swagger.v3.oas.annotations.media.Schema; public record FileExchangeExportRequest( @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") List dossierIds, @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") List fileIds, - @Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS and NER_ENTITIES file will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles, + @Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles, @Schema(description = "If set to true, the MANUAL_REDACTIONS file will be excluded from the export.", defaultValue = "false") boolean excludeManualRedactions, @Schema(description = "If set to true, the ENTITY_LOG and COMPONENT_LOG (with its overrides) will be excluded from the export.", defaultValue = "false") boolean excludeAnalysisLogs, @Schema(description = "If set to true, the File Attributes will be excluded from the export.", defaultValue = "false") boolean excludeFileAttributes, From 999ecafdf8b6bd83cd2c36e32cf8a25d31441929 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:21:34 +0200 Subject: [PATCH 03/13] RED-9255: implement file exchange * add simplified text file to layoutparsing file export * add validation for fileId/dossierId combinations --- .../processor/service/ReanalysisService.java | 3 +-- .../v1/api/shared/model/FileStatusFilter.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 2edbef718..d9c8e81e5 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -270,9 +270,8 @@ public class ReanalysisService { fileStatusService.getDossierTemplateStatus(dossierTemplateId) .stream() - .filter(file -> !file.isSoftOrHardDeleted()) .filter(file -> isInList(file, reAnalysisSettings)) - .filter(file -> file.getProcessingStatus().equals(ProcessingStatus.ERROR) || !reAnalysisSettings.onlyErrorFiles()) + .filter(reAnalysisSettings.fileStatusFilter().fileStatusPredicate()) .peek(file -> log.info("Reanalyzing file {}", file.getId())) .forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reAnalysisSettings.repeatStructureAnalysis())); diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java index 60768e994..0e8b705c9 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java @@ -2,7 +2,9 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -29,4 +31,21 @@ public class FileStatusFilter { this.includeHardDeletedFiles = false; } + + public Predicate fileStatusPredicate() { + + if (this.getProcessingStatusList() == null) { + this.setProcessingStatusList(new ArrayList<>()); + } + + if (this.getWorkflowStatusList() == null) { + this.setWorkflowStatusList(new ArrayList<>()); + } + + return fileStatus -> (this.getProcessingStatusList().isEmpty() || this.getProcessingStatusList().contains(fileStatus.getProcessingStatus())) + && (this.getWorkflowStatusList().isEmpty() || this.getWorkflowStatusList().contains(fileStatus.getWorkflowStatus())) + && (this.isIncludeSoftDeletedFiles() || fileStatus.getDeleted() == null) + && (this.isIncludeHardDeletedFiles() || fileStatus.getHardDeletedTime() == null); + } + } From cdece930370d18ff15d207e111175465fea5b3df Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:22:39 +0200 Subject: [PATCH 04/13] RED-9255: implement file exchange * add simplified text file to layoutparsing file export * add validation for fileId/dossierId combinations --- .../api/impl/controller/SupportController.java | 8 ++++---- .../v1/api/external/resource/SupportResource.java | 4 ++-- .../v1/processor/service/ReanalysisService.java | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index bf6dd6756..14429ea88 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -23,7 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper; import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService; -import com.iqser.red.service.persistence.service.v1.api.external.model.ReAnalysisSettings; +import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; @@ -48,10 +48,10 @@ public class SupportController implements SupportResource { @Override - public void reanalyzeFiles(String dossierTemplateId, ReAnalysisSettings reAnalysisSettings) { + public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { - log.info("Scheduling reanalysis for all files in Dossier Tempalte {} with settings {}", dossierTemplateId, reAnalysisSettings); - reanalysisService.reanalyzeTemplate(dossierTemplateId, reAnalysisSettings); + log.info("Scheduling reanalysis for all files in Dossier Tempalte {} with settings {}", dossierTemplateId, reanalysisSettings); + reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings); } diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index 8f89b665b..632ad9b5c 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -13,7 +13,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.multipart.MultipartFile; -import com.iqser.red.service.persistence.service.v1.api.external.model.ReAnalysisSettings; +import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter; @@ -55,7 +55,7 @@ public interface SupportResource { @Operation(summary = "Reanalyze all files in dossier template", description = "None") @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")}) void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, - @RequestBody ReAnalysisSettings reAnalysisSettings); + @RequestBody ReanalysisSettings reAnalysisSettings); @PostMapping(value = ERROR_REANALYSIS_REST_PATH) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index d9c8e81e5..ef31c7062 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -9,7 +9,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.external.model.ReAnalysisSettings; +import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -266,19 +266,19 @@ public class ReanalysisService { } - public void reanalyzeTemplate(String dossierTemplateId, ReAnalysisSettings reAnalysisSettings) { + public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { fileStatusService.getDossierTemplateStatus(dossierTemplateId) .stream() - .filter(file -> isInList(file, reAnalysisSettings)) - .filter(reAnalysisSettings.fileStatusFilter().fileStatusPredicate()) + .filter(file -> isInList(file, reanalysisSettings)) + .filter(reanalysisSettings.fileStatusFilter().fileStatusPredicate()) .peek(file -> log.info("Reanalyzing file {}", file.getId())) - .forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reAnalysisSettings.repeatStructureAnalysis())); + .forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis())); } - private boolean isInList(FileModel file, ReAnalysisSettings reAnalysisSettings) { + private boolean isInList(FileModel file, ReanalysisSettings reAnalysisSettings) { return (reAnalysisSettings.fileIds().isEmpty() || reAnalysisSettings.fileIds().contains(file.getId())) // && (reAnalysisSettings.dossierIds().isEmpty() || reAnalysisSettings.dossierIds().contains(file.getDossierId())); From 46f9523b2cc18f3df3b6647c7caccd76feeef42e Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:40:52 +0200 Subject: [PATCH 05/13] RED-9255: implement file exchange * add simplified text file to layoutparsing file export * add validation for fileId/dossierId combinations --- .../impl/controller/SupportController.java | 1 - .../DossierTemplateControllerV2.java | 13 ++-- .../external/resource/SupportResource.java | 19 +++++- .../service/FileExchangeExportService.java | 41 ++----------- .../DossierIdFileIdRequestValidator.java | 59 +++++++++++++++++++ .../processor/service/ReanalysisService.java | 5 ++ .../model/FileExchangeExportRequest.java | 5 +- 7 files changed, 95 insertions(+), 48 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index 14429ea88..819120bb2 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -50,7 +50,6 @@ public class SupportController implements SupportResource { @Override public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { - log.info("Scheduling reanalysis for all files in Dossier Tempalte {} with settings {}", dossierTemplateId, reanalysisSettings); reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings); } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java index 465135712..c7a56bc04 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java @@ -56,6 +56,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest; import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; @@ -216,12 +217,12 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { Path mappingFile = saveToFile(file); String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); - com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, - nameToUse, - fileName, - delimiter, - encoding, - mappingFile.toFile()); + ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, + nameToUse, + fileName, + delimiter, + encoding, + mappingFile.toFile()); Files.deleteIfExists(mappingFile); diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index 632ad9b5c..fb4456550 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -51,8 +51,23 @@ public interface SupportResource { String IMPORT = "/import"; - @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE) - @Operation(summary = "Reanalyze all files in dossier template", description = "None") + @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE) + @Operation(summary = "Reanalyze all files in dossier template", description = """ + ## Reanalyze Files Endpoint + + Use this endpoint to reanalyze all files in a specified Dossier Template. The reanalysis process can be tailored using various filtering options provided in the request body. + + ### Parameters + + - **DossierTemplateId**: Specifies the Dossier Template whose files need to be reanalyzed. + + ### Request Body Configuration Options + + - **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected for reanalysis. + - **fileIds**: List of file IDs to filter. If empty, all files are selected for reanalysis. + - **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated. + - **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null. + """) @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")}) void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reAnalysisSettings); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java index 223a89ccd..b2980bd15 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java @@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.management.v1.processor.dataexchange.Ex import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService; @@ -61,6 +62,8 @@ public class FileExchangeExportService { ObjectMapper mapper; RabbitTemplate rabbitTemplate; EntityTypeExportService entityTypeExportService; + DossierIdFileIdRequestValidator requestValidator; + @Observed(name = "FileExchangeExportService", contextualName = "prepare-export") public DownloadResponse prepareExportDownload(String dossierTemplateId, FileExchangeExportRequest request) { @@ -69,7 +72,7 @@ public class FileExchangeExportService { DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId); String dossierTemplateName = dossierTemplate.getName(); - validateRequest(dossierTemplate, request); + requestValidator.validateRequestOrThrow404(dossierTemplateId, request.dossierIds(), request.fileIds()); String downloadFilename = dossierTemplateName + "_file-exchange.zip"; String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_file-exchange"); @@ -87,42 +90,6 @@ public class FileExchangeExportService { return new DownloadResponse(storageId); } - @Observed(name = "FileExchangeExportService", contextualName = "validate-request") - private void validateRequest(DossierTemplateEntity dossierTemplate, FileExchangeExportRequest request) { - - Set dossierIds = new HashSet<>(request.dossierIds()); - if (!request.dossierIds().isEmpty()) { - Set availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId()); - Set nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds); - if (!nonAvailableDossiers.isEmpty()) { - throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", - String.join(", ", nonAvailableDossiers), - dossierTemplate.getId())); - } - } - - if (!request.fileIds().isEmpty()) { - - Set availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplate.getId()) - .stream() - .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId())) - .map(FileModel::getId) - .collect(Collectors.toSet()); - - Set nonAvailableFiles = Sets.difference(new HashSet<>(request.fileIds()), availableFileIds); - - if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) { - throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplate.getId())); - } - - if (!nonAvailableFiles.isEmpty()) { - throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s", - String.join(", ", nonAvailableFiles), - String.join(", ", dossierIds), - dossierTemplate.getId())); - } - } - } private void addToExportDownloadQueue(ExportDownloadMessage downloadJob) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java new file mode 100644 index 000000000..11dc6aa41 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java @@ -0,0 +1,59 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.google.common.collect.Sets; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; + +import io.micrometer.observation.annotation.Observed; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DossierIdFileIdRequestValidator { + + DossierManagementService dossierManagementService; + FileStatusManagementService fileStatusManagementService; + + @Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request") + public void validateRequestOrThrow404(String dossierTemplateId, Set dossierIds, Set fileIds) { + + if (!dossierIds.isEmpty()) { + Set availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId); + Set nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds); + if (!nonAvailableDossiers.isEmpty()) { + throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId)); + } + } + + if (!fileIds.isEmpty()) { + Set availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId) + .stream() + .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId())) + .map(FileModel::getId) + .collect(Collectors.toSet()); + + Set nonAvailableFiles = Sets.difference(fileIds, availableFileIds); + + if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) { + throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplateId)); + } + + if (!nonAvailableFiles.isEmpty()) { + throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s", + String.join(", ", nonAvailableFiles), + String.join(", ", dossierIds), + dossierTemplateId)); + } + } + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index ef31c7062..daf1afdf3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -5,11 +5,13 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -26,6 +28,7 @@ import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Predicate; @@ -41,6 +44,7 @@ public class ReanalysisService { private final IndexingService indexingService; private final PDFTronClient pDFTronRedactionClient; private final FileManagementStorageService fileManagementStorageService; + private final DossierIdFileIdRequestValidator requestValidator; private final Predicate validFilesFilter = fileStatus -> !fileStatus.isSoftOrHardDeleted() && !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED); private final Predicate errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR); @@ -268,6 +272,7 @@ public class ReanalysisService { public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { + requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds()); fileStatusService.getDossierTemplateStatus(dossierTemplateId) .stream() .filter(file -> isInList(file, reanalysisSettings)) diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java index 15783d39e..d4e4ff97e 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java @@ -1,12 +1,13 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.util.List; +import java.util.Set; import io.swagger.v3.oas.annotations.media.Schema; public record FileExchangeExportRequest( - @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") List dossierIds, - @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") List fileIds, + @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") Set dossierIds, + @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") Set fileIds, @Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles, @Schema(description = "If set to true, the MANUAL_REDACTIONS file will be excluded from the export.", defaultValue = "false") boolean excludeManualRedactions, @Schema(description = "If set to true, the ENTITY_LOG and COMPONENT_LOG (with its overrides) will be excluded from the export.", defaultValue = "false") boolean excludeAnalysisLogs, From fbf9320f6cf9ee7e23043e36f5cefa6fdbcf3de8 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:42:51 +0200 Subject: [PATCH 06/13] RED-9255: implement file exchange * add simplified text file to layoutparsing file export * add validation for fileId/dossierId combinations --- .../v1/external/api/impl/controller/SupportController.java | 2 +- .../service/v1/api/external/resource/SupportResource.java | 5 ++--- .../management/v1/processor/service/ReanalysisService.java | 5 +---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index 819120bb2..1dd6424b7 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -23,7 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper; import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService; -import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index fb4456550..fed505a12 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -13,7 +13,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.multipart.MultipartFile; -import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter; @@ -69,8 +69,7 @@ public interface SupportResource { - **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null. """) @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")}) - void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, - @RequestBody ReanalysisSettings reAnalysisSettings); + void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings); @PostMapping(value = ERROR_REANALYSIS_REST_PATH) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index daf1afdf3..7b18c51f6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -5,13 +5,11 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings; -import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -28,7 +26,6 @@ import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Predicate; From 4b61a19a95eb7994fea0a563b8921725e34085b1 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:46:13 +0200 Subject: [PATCH 07/13] RED-9255: implement file exchange * add simplified text file to layoutparsing file export * add validation for fileId/dossierId combinations --- .../processor/service/ReanalysisService.java | 24 ++++++++++++++++--- .../v1/api/shared/model/FileStatusFilter.java | 2 +- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 7b18c51f6..da097e7fc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -270,13 +270,31 @@ public class ReanalysisService { public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds()); - fileStatusService.getDossierTemplateStatus(dossierTemplateId) + + var files = fileStatusService.getDossierTemplateStatus(dossierTemplateId) .stream() .filter(file -> isInList(file, reanalysisSettings)) - .filter(reanalysisSettings.fileStatusFilter().fileStatusPredicate()) + .filter(reanalysisSettings.fileStatusFilter().asPredicate()) .peek(file -> log.info("Reanalyzing file {}", file.getId())) - .forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis())); + .toList(); + validateFilesForReanalysis(files); + + files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis())); + + } + + + private void validateFilesForReanalysis(List files) { + + for (var file : files) { + if (file.isSoftOrHardDeleted()) { + throw new BadRequestException("Cannot reanalyse deleted file!"); + } + if (file.getWorkflowStatus() == WorkflowStatus.APPROVED) { + throw new BadRequestException("Cannot reanalyse approved file!"); + } + } } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java index 0e8b705c9..7c2d24589 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java @@ -32,7 +32,7 @@ public class FileStatusFilter { } - public Predicate fileStatusPredicate() { + public Predicate asPredicate() { if (this.getProcessingStatusList() == null) { this.setProcessingStatusList(new ArrayList<>()); From e382e4e833a303e0af82319e5b17ac887da0ea78 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:55:48 +0200 Subject: [PATCH 08/13] RED-9255: implement file exchange * add simplified text file to layoutparsing file export * add validation for fileId/dossierId combinations --- .../api/shared/model/ReanalysisSettings.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java new file mode 100644 index 000000000..da05515b3 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model; + +import java.util.Optional; +import java.util.Set; + +import io.swagger.v3.oas.annotations.media.Schema; + +public record ReanalysisSettings( + @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set dossierIds, + @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set fileIds, + @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis, + @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "{}") FileStatusFilter fileStatusFilter +) { + + public FileStatusFilter fileStatusFilter() { + + return Optional.ofNullable(fileStatusFilter) + .orElse(new FileStatusFilter()); + } + +} From 5dd27f682b22b28f1361825b1e06c8f4324a9206 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 11:57:18 +0200 Subject: [PATCH 09/13] RED-9375: fix component mapping endpoint validation --- .../DossierTemplateControllerV2.java | 45 ++++++++++--------- .../resource/DossierTemplateResource.java | 2 +- .../service/ComponentMappingService.java | 8 ++-- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java index c7a56bc04..9793ff987 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java @@ -204,29 +204,32 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { @Override @SneakyThrows @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") - public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) { + public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) { dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name; + String nameToUse = Strings.isNullOrEmpty(name) ? file.getOriginalFilename() : name; if (Strings.isNullOrEmpty(nameToUse)) { throw new BadRequestException("The provided file name is not valid!"); } + if (Strings.isNullOrEmpty(delimiter) || delimiter.length() != 1) { + throw new BadRequestException("The provided delimiter is not valid! Only a single character is allowed."); + } + char cleanDelimiter = delimiter.charAt(0); + Path mappingFile = saveToFile(file); - String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); + try { + String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); - ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, - nameToUse, - fileName, - delimiter, - encoding, - mappingFile.toFile()); + ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile()); - Files.deleteIfExists(mappingFile); + return componentMappingMapper.toModel(metaData); + } finally { + Files.deleteIfExists(mappingFile); + } - return componentMappingMapper.toModel(metaData); } @@ -239,15 +242,13 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { Path mappingFile = saveToFile(file); - com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId, - componentMappingId, - encoding, - delimiter, - mappingFile.toFile()); + try { + ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId, componentMappingId, encoding, delimiter, mappingFile.toFile()); - Files.deleteIfExists(mappingFile); - - return componentMappingMapper.toModel(resultMetaData); + return componentMappingMapper.toModel(resultMetaData); + } finally { + Files.deleteIfExists(mappingFile); + } } @@ -279,7 +280,11 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - componentMappingService.delete(dossierTemplateId, componentMappingId); + try { + componentMappingService.delete(dossierTemplateId, componentMappingId); + } catch (Exception ignored) { + + } return new ResponseEntity<>(HttpStatus.NO_CONTENT); } diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java index 15a9ce5e9..0283e562a 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java @@ -140,7 +140,7 @@ public interface DossierTemplateResource { @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name, @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, - @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter); @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None") diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java index 82400e741..845f21b6c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java @@ -124,7 +124,7 @@ public class ComponentMappingService { } - private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws CsvException, BadRequestException, IOException { + private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws BadRequestException, IOException { Path tempFile = Files.createTempFile("mapping", ".tmp"); @@ -150,10 +150,12 @@ public class ComponentMappingService { } catch (IOException e) { throw new BadRequestException("Error while sorting the csv file", e); + } catch (CsvException e) { + throw new BadRequestException("The file could not be parsed as a csv file", e); + } finally { + Files.deleteIfExists(tempFile); } - Files.deleteIfExists(tempFile); - return new CsvStats(Arrays.asList(columnLabels), numberOfLines); } From 1a585e9fe3227765e6244f111ca37797ae8aa52b Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 13:42:29 +0200 Subject: [PATCH 10/13] RED-9375: fix component mapping endpoint validation --- .../v1/processor/service/ComponentMappingService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java index 845f21b6c..206632df3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java @@ -140,6 +140,10 @@ public class ComponentMappingService { columnLabels = rows.remove(0); // remove header row + if (Arrays.stream(columnLabels).distinct().count() < columnLabels.length) { + throw new BadRequestException("Column labels may not contain duplicates!"); + } + numberOfLines = (int) reader.getLinesRead() - 1; // subtract header row rows.sort(CSV_SORTER); From 23929d277291381680fee9c45dabe727c81f0475 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 13:43:20 +0200 Subject: [PATCH 11/13] RED-9375: fix component mapping endpoint validation --- .../server/integration/tests/FileExchangeImportExportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java index 70487c313..91b512eac 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java @@ -123,7 +123,7 @@ public class FileExchangeImportExportTest extends AbstractPersistenceServerServi private FileExchangeExportRequest buildDefaultRequest() { - return new FileExchangeExportRequest(Collections.emptyList(), Collections.emptyList(), false, false, false, false, true); + return new FileExchangeExportRequest(Collections.emptySet(), Collections.emptySet(), false, false, false, false, true); } From a1b224adbdb491c7e047e90ebdd5fbb27c136ca1 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 13:53:51 +0200 Subject: [PATCH 12/13] RED-9375: fix component mapping endpoint validation --- .../dataexchange/service/FileExchangeExportService.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java index b2980bd15..49aa65dfb 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java @@ -91,7 +91,6 @@ public class FileExchangeExportService { } - private void addToExportDownloadQueue(ExportDownloadMessage downloadJob) { rabbitTemplate.convertAndSend(MessagingConfiguration.EXPORT_DOWNLOAD_QUEUE, downloadJob, message -> { @@ -133,11 +132,10 @@ public class FileExchangeExportService { List files = fileStatusManagementService.getDossierStatus(dossier.getId()); - if (!request.fileIds().isEmpty() // - && !request.dossierIds().isEmpty() // + if (!request.dossierIds().contains(dossier.getId()) // && files.stream() - .noneMatch(fileModel -> request.fileIds().contains(fileModel.getId()))) { - // no files of dossier in requested files and dossier not explicitly requested -> don't export it. + .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) { + // dossier has no files in requested files and dossier not explicitly requested -> don't export it. return; } From 07a0bacf557f7baf6463ce67d1064906c02df733 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Jun 2024 14:31:05 +0200 Subject: [PATCH 13/13] RED-9375: fix component mapping endpoint validation --- .../v1/external/api/impl/controller/ReanalysisController.java | 1 + .../service/v1/api/external/resource/SupportResource.java | 2 +- .../management/v1/processor/service/ReanalysisService.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java index 666ab8e7e..25ba16d58 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java @@ -46,6 +46,7 @@ public class ReanalysisController implements ReanalysisResource { public void reanalyzeDossier(@PathVariable(DOSSIER_ID) String dossierId, @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force) { accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId); + reanalysisService.reanalyzeDossier(dossierId, force); auditPersistenceService.audit(AuditRequest.builder() diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index fed505a12..858956d60 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -51,7 +51,7 @@ public interface SupportResource { String IMPORT = "/import"; - @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE) + @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE) @Operation(summary = "Reanalyze all files in dossier template", description = """ ## Reanalyze Files Endpoint diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index da097e7fc..fea636eb6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -56,7 +56,7 @@ public class ReanalysisService { public void reanalyzeDossier(String dossierId, boolean force) { - var relevantFiles = getAllFilesForDossier(dossierId, validFilesFilter); + List relevantFiles = getAllFilesForDossier(dossierId, validFilesFilter); reanalyseFiles(force, relevantFiles); }