diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java index 666ab8e7e..25ba16d58 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/ReanalysisController.java @@ -46,6 +46,7 @@ public class ReanalysisController implements ReanalysisResource { public void reanalyzeDossier(@PathVariable(DOSSIER_ID) String dossierId, @RequestParam(value = FORCE_PARAM, required = false, defaultValue = FALSE) boolean force) { accessControlService.checkDossierExistenceAndAccessPermissionsToDossier(dossierId); + reanalysisService.reanalyzeDossier(dossierId, force); auditPersistenceService.audit(AuditRequest.builder() diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java index 80bbdc251..1dd6424b7 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java @@ -23,6 +23,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.FileSta import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusMapper; import com.iqser.red.service.persistence.management.v1.processor.service.ReanalysisService; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.FileExchangeExportService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.external.resource.SupportResource; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; @@ -47,9 +48,9 @@ public class SupportController implements SupportResource { @Override - public void reanalyzeFiles(String dossierTemplateId, boolean repeatStructureAnalysis) { + public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { - reanalysisService.reanalyzeTemplate(dossierTemplateId, repeatStructureAnalysis); + reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings); } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java index 465135712..9793ff987 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java @@ -56,6 +56,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionUpdateRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest; import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel; @@ -203,29 +204,32 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { @Override @SneakyThrows @PreAuthorize("hasAuthority('" + WRITE_RULES + "')") - public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, char delimiter) { + public ComponentMappingMetadataModel uploadMapping(String dossierTemplateId, MultipartFile file, String name, String encoding, String delimiter) { dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - String nameToUse = Strings.isNullOrEmpty(name) ? file.getName().split("\\.")[0] : name; + String nameToUse = Strings.isNullOrEmpty(name) ? file.getOriginalFilename() : name; if (Strings.isNullOrEmpty(nameToUse)) { throw new BadRequestException("The provided file name is not valid!"); } + if (Strings.isNullOrEmpty(delimiter) || delimiter.length() != 1) { + throw new BadRequestException("The provided delimiter is not valid! Only a single character is allowed."); + } + char cleanDelimiter = delimiter.charAt(0); + Path mappingFile = saveToFile(file); - String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); + try { + String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename(); - com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, - nameToUse, - fileName, - delimiter, - encoding, - mappingFile.toFile()); + ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId, nameToUse, fileName, cleanDelimiter, encoding, mappingFile.toFile()); - Files.deleteIfExists(mappingFile); + return componentMappingMapper.toModel(metaData); + } finally { + Files.deleteIfExists(mappingFile); + } - return componentMappingMapper.toModel(metaData); } @@ -238,15 +242,13 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { Path mappingFile = saveToFile(file); - com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId, - componentMappingId, - encoding, - delimiter, - mappingFile.toFile()); + try { + ComponentMappingMetadata resultMetaData = componentMappingService.update(dossierTemplateId, componentMappingId, encoding, delimiter, mappingFile.toFile()); - Files.deleteIfExists(mappingFile); - - return componentMappingMapper.toModel(resultMetaData); + return componentMappingMapper.toModel(resultMetaData); + } finally { + Files.deleteIfExists(mappingFile); + } } @@ -278,7 +280,11 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource { dossierTemplatePersistenceService.checkDossierTemplateExistsOrElseThrow404(dossierTemplateId); - componentMappingService.delete(dossierTemplateId, componentMappingId); + try { + componentMappingService.delete(dossierTemplateId, componentMappingId); + } catch (Exception ignored) { + + } return new ResponseEntity<>(HttpStatus.NO_CONTENT); } diff --git a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java index 1a8b4f61f..858956d60 100644 --- a/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java +++ b/persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java @@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.DownloadResponse; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileStatusFilter; @@ -51,10 +52,24 @@ public interface SupportResource { @PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE) - @Operation(summary = "Reanalyze all files in dossier template", description = "None") + @Operation(summary = "Reanalyze all files in dossier template", description = """ + ## Reanalyze Files Endpoint + + Use this endpoint to reanalyze all files in a specified Dossier Template. The reanalysis process can be tailored using various filtering options provided in the request body. + + ### Parameters + + - **DossierTemplateId**: Specifies the Dossier Template whose files need to be reanalyzed. + + ### Request Body Configuration Options + + - **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected for reanalysis. + - **fileIds**: List of file IDs to filter. If empty, all files are selected for reanalysis. + - **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated. + - **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null. + """) @ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")}) - void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, - @RequestParam(value = FULL_REANALYSIS_PARAM, required = false, defaultValue = FALSE) boolean repeatStructureAnalysis); + void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId, @RequestBody ReanalysisSettings reanalysisSettings); @PostMapping(value = ERROR_REANALYSIS_REST_PATH) @@ -115,7 +130,7 @@ public interface SupportResource { - **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected. - **fileIds**: List of file IDs to filter. If empty, all files are selected. - - **excludeLayoutFiles**: Boolean. If true, excludes DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS and NER_ENTITIES files. + - **excludeLayoutFiles**: Boolean. If true, excludes DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files. - **excludeManualRedactions**: Boolean. If true, excludes MANUAL_REDACTIONS files. - **excludeAnalysisLogs**: Boolean. If true, excludes ENTITY_LOG and COMPONENT_LOG (with overrides) files. - **excludeFileAttributes**: Boolean. If true, excludes file attributes. @@ -127,7 +142,7 @@ public interface SupportResource { @ResponseStatus(value = HttpStatus.NO_CONTENT) @ResponseBody @PostMapping(value = FILE_EXCHANGE_REST_PATH + IMPORT, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files. If the request parameter dossierTemplateId is set, the files will be imported into the existing DossierTemplate.") + @Operation(summary = "Imports a file exchange export zip.", description = "Use this endpoint to import a full export of a given Dossier Template including all its configurations, dossiers, and files.") @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "OK")}) void importFiles(@RequestPart(name = "file") MultipartFile file); diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java index 15a9ce5e9..0283e562a 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/java/com/iqser/red/service/persistence/service/v2/api/external/resource/DossierTemplateResource.java @@ -140,7 +140,7 @@ public interface DossierTemplateResource { @Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, @Parameter(name = MAPPING_NAME_PARAM, description = "String of what the mapping should be accessible under. If left empty, the name of the file without the ending will be used as name.") @RequestParam(value = MAPPING_NAME_PARAM, required = false, defaultValue = "") String name, @Parameter(name = ENCODING_PARAM, description = "The encoding of the file. Default is UTF-8.") @RequestParam(value = ENCODING_PARAM, required = false, defaultValue = "UTF-8") String encoding, - @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") char delimiter); + @Parameter(name = DELIMITER_PARAM, description = "The delimiter used in the file. Default is ','") @RequestParam(value = DELIMITER_PARAM, required = false, defaultValue = ",") String delimiter); @Operation(summary = "Update an existing component mapping of a DossierTemplate.", description = "None") diff --git a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml index ab41b3627..765dd7e1f 100644 --- a/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml +++ b/persistence-service-v1/persistence-service-external-api-v2/src/main/resources/api/documine.yaml @@ -594,7 +594,7 @@ paths: - Component Definitions summary: Create new component definitions description: | - Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case + Create new component definitions for a given dossier template. The component will have a technical name which is automatically converted to snake case that can't be updated after the creation. The rank is used to determine the order in which the components are displayed. The component's rank will automatically be appended at the end based on the current number of components of this dossier template. parameters: diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java index 8237ca925..1c3d948ea 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/FileExchangeNames.java @@ -24,6 +24,7 @@ public class FileExchangeNames { public static Definition PAGES = new Definition(FileType.DOCUMENT_PAGES); public static Definition TEXT = new Definition(FileType.DOCUMENT_TEXT); public static Definition POSITIONS = new Definition(FileType.DOCUMENT_POSITION); + public static Definition SIMPLIFIED_TEXT = new Definition(FileType.SIMPLIFIED_TEXT); public static Definition NER_ENTITIES = new Definition(FileType.NER_ENTITIES); public static Definition TABLES = new Definition(FileType.TABLES); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java index 390a95ae3..49aa65dfb 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java @@ -3,18 +3,24 @@ package com.iqser.red.service.persistence.management.v1.processor.dataexchange.s import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import com.iqser.red.service.persistence.management.v1.processor.configuration.MessagingConfiguration; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.ExportDownloadMessage; import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService; +import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService; @@ -56,12 +62,18 @@ public class FileExchangeExportService { ObjectMapper mapper; RabbitTemplate rabbitTemplate; EntityTypeExportService entityTypeExportService; + DossierIdFileIdRequestValidator requestValidator; + @Observed(name = "FileExchangeExportService", contextualName = "prepare-export") public DownloadResponse prepareExportDownload(String dossierTemplateId, FileExchangeExportRequest request) { var mimeType = "application/zip"; - String dossierTemplateName = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId).getName(); + DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId); + String dossierTemplateName = dossierTemplate.getName(); + + requestValidator.validateRequestOrThrow404(dossierTemplateId, request.dossierIds(), request.fileIds()); + String downloadFilename = dossierTemplateName + "_file-exchange.zip"; String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_file-exchange"); @@ -118,14 +130,25 @@ public class FileExchangeExportService { @Observed(name = "FileExchangeExportService", contextualName = "export-dossier") public void addDossierToArchive(FileSystemBackedArchiver archiver, Path folder, FileExchangeExportRequest request, Dossier dossier) { + List files = fileStatusManagementService.getDossierStatus(dossier.getId()); + + if (!request.dossierIds().contains(dossier.getId()) // + && files.stream() + .noneMatch(fileModel -> request.fileIds().isEmpty() || request.fileIds().contains(fileModel.getId()))) { + // dossier has no files in requested files and dossier not explicitly requested -> don't export it. + return; + } + Path dossierFolder = folder.resolve(dossier.getId()); archiver.addEntry(new FileSystemBackedArchiver.ArchiveModel(dossierFolder, FileExchangeNames.DOSSIER, mapper.writeValueAsBytes(dossier))); - for (FileModel fileEntity : fileStatusManagementService.getDossierStatus(dossier.getId())) { + + for (FileModel fileEntity : files) { if (!request.fileIds().isEmpty() && !request.fileIds().contains(fileEntity.getId())) { continue; } addFileToArchive(archiver, dossierFolder, request, fileEntity); } + List types = dictionaryPersistenceService.getAllTypesForDossier(dossier.getId(), false); String entitiesFolder = dossierFolder.resolve(FileExchangeNames.DOSSIER_ENTITY_FOLDER).toFile().toString(); @@ -164,6 +187,7 @@ public class FileExchangeExportService { addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.POSITIONS); addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.PAGES); addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.NER_ENTITIES); + addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.SIMPLIFIED_TEXT); } addArchiveModelForStorageFile(archiver, file, fileFolder, FileExchangeNames.FIGURE); @@ -220,6 +244,7 @@ public class FileExchangeExportService { archiver.addEntry(archiveModel); } + @Observed(name = "FileExchangeExportService", contextualName = "store-archive") private void storeZipFile(String storageId, FileSystemBackedArchiver fileSystemBackedArchiver) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java index a1a1db76c..d3ef43887 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/zipreaders/FileExchangeArchiveReader.java @@ -97,6 +97,7 @@ public class FileExchangeArchiveReader { FileExchangeNames.PAGES, FileExchangeNames.TEXT, FileExchangeNames.POSITIONS, + FileExchangeNames.SIMPLIFIED_TEXT, FileExchangeNames.NER_ENTITIES, FileExchangeNames.TABLES, FileExchangeNames.IMAGES, diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java index 82400e741..206632df3 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ComponentMappingService.java @@ -124,7 +124,7 @@ public class ComponentMappingService { } - private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws CsvException, BadRequestException, IOException { + private static CsvStats sortCSVFile(char delimiter, File mappingFile, Charset charset) throws BadRequestException, IOException { Path tempFile = Files.createTempFile("mapping", ".tmp"); @@ -140,6 +140,10 @@ public class ComponentMappingService { columnLabels = rows.remove(0); // remove header row + if (Arrays.stream(columnLabels).distinct().count() < columnLabels.length) { + throw new BadRequestException("Column labels may not contain duplicates!"); + } + numberOfLines = (int) reader.getLinesRead() - 1; // subtract header row rows.sort(CSV_SORTER); @@ -150,10 +154,12 @@ public class ComponentMappingService { } catch (IOException e) { throw new BadRequestException("Error while sorting the csv file", e); + } catch (CsvException e) { + throw new BadRequestException("The file could not be parsed as a csv file", e); + } finally { + Files.deleteIfExists(tempFile); } - Files.deleteIfExists(tempFile); - return new CsvStats(Arrays.asList(columnLabels), numberOfLines); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java new file mode 100644 index 000000000..11dc6aa41 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java @@ -0,0 +1,59 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.google.common.collect.Sets; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; + +import io.micrometer.observation.annotation.Observed; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Service +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class DossierIdFileIdRequestValidator { + + DossierManagementService dossierManagementService; + FileStatusManagementService fileStatusManagementService; + + @Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request") + public void validateRequestOrThrow404(String dossierTemplateId, Set dossierIds, Set fileIds) { + + if (!dossierIds.isEmpty()) { + Set availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId); + Set nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds); + if (!nonAvailableDossiers.isEmpty()) { + throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId)); + } + } + + if (!fileIds.isEmpty()) { + Set availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId) + .stream() + .filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId())) + .map(FileModel::getId) + .collect(Collectors.toSet()); + + Set nonAvailableFiles = Sets.difference(fileIds, availableFileIds); + + if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) { + throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplateId)); + } + + if (!nonAvailableFiles.isEmpty()) { + throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s", + String.join(", ", nonAvailableFiles), + String.join(", ", dossierIds), + dossierTemplateId)); + } + } + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java index 130317c51..7ef356635 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java @@ -119,6 +119,13 @@ public class DossierManagementService { } + @Transactional + public Set getAllDossierIdsForDossierTemplateId(String dossierTemplateId) { + + return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId); + + } + public DossierInformation getDossierInformation(List filteredDossierIds) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java index 0b69d3073..b7ddd096b 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierService.java @@ -1,5 +1,11 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Set; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -17,12 +23,6 @@ import jakarta.validation.ConstraintViolationException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.util.List; -import java.util.Set; - /** * Provides the internal interface between dossier request and the actual persistence. */ @@ -143,6 +143,11 @@ public class DossierService { return dossierPersistenceService.findAllDossiersForDossierTemplateId(dossierTemplateId); } + public Set getAllDossierIdsForDossierTemplateId(String dossierTemplateId) { + + return dossierPersistenceService.findAllDossierIdsForDossierTemplateId(dossierTemplateId); + } + public Set changesSince(OffsetDateTime since) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java index 4e956f861..fea636eb6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java @@ -9,6 +9,7 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.ReanalysisSettings; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; @@ -40,6 +41,7 @@ public class ReanalysisService { private final IndexingService indexingService; private final PDFTronClient pDFTronRedactionClient; private final FileManagementStorageService fileManagementStorageService; + private final DossierIdFileIdRequestValidator requestValidator; private final Predicate validFilesFilter = fileStatus -> !fileStatus.isSoftOrHardDeleted() && !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED); private final Predicate errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR); @@ -54,7 +56,7 @@ public class ReanalysisService { public void reanalyzeDossier(String dossierId, boolean force) { - var relevantFiles = getAllFilesForDossier(dossierId, validFilesFilter); + List relevantFiles = getAllFilesForDossier(dossierId, validFilesFilter); reanalyseFiles(force, relevantFiles); } @@ -265,13 +267,41 @@ public class ReanalysisService { } - public void reanalyzeTemplate(String dossierTemplateId, boolean repeatStructureAnalysis) { + public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) { - fileStatusService.getDossierTemplateStatus(dossierTemplateId) + requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds()); + + var files = fileStatusService.getDossierTemplateStatus(dossierTemplateId) .stream() - .filter(file -> !file.isSoftOrHardDeleted()) - .forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, repeatStructureAnalysis)); + .filter(file -> isInList(file, reanalysisSettings)) + .filter(reanalysisSettings.fileStatusFilter().asPredicate()) + .peek(file -> log.info("Reanalyzing file {}", file.getId())) + .toList(); + + validateFilesForReanalysis(files); + + files.forEach(file -> fileStatusService.setStatusFullReprocess(file.getDossierId(), file.getId(), false, reanalysisSettings.repeatStructureAnalysis())); } + + private void validateFilesForReanalysis(List files) { + + for (var file : files) { + if (file.isSoftOrHardDeleted()) { + throw new BadRequestException("Cannot reanalyse deleted file!"); + } + if (file.getWorkflowStatus() == WorkflowStatus.APPROVED) { + throw new BadRequestException("Cannot reanalyse approved file!"); + } + } + } + + + private boolean isInList(FileModel file, ReanalysisSettings reAnalysisSettings) { + + return (reAnalysisSettings.fileIds().isEmpty() || reAnalysisSettings.fileIds().contains(file.getId())) // + && (reAnalysisSettings.dossierIds().isEmpty() || reAnalysisSettings.dossierIds().contains(file.getDossierId())); + } + } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java index 3046c748c..a4e10a712 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierPersistenceService.java @@ -195,6 +195,12 @@ public class DossierPersistenceService { } + public Set findAllDossierIdsForDossierTemplateId(String dossierTemplateId) { + + return dossierRepository.findIdsByDossierTemplateId(dossierTemplateId); + } + + @Transactional public void hardDelete(String dossierId) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java index 667d5e7c4..247d3c8bc 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierRepository.java @@ -2,6 +2,7 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persis import java.time.OffsetDateTime; import java.util.List; +import java.util.Set; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; @@ -84,6 +85,10 @@ public interface DossierRepository extends JpaRepository List findByDossierTemplateId(@Param("dossierTemplateId") String dossierTemplateId); + @Query("select d.id from DossierEntity d where d.dossierTemplateId = :dossierTemplateId") + Set findIdsByDossierTemplateId(String dossierTemplateId); + + @Modifying @Query("update DossierEntity d set d.watermarkId = null where d.watermarkId = :watermarkId") int countDeleteWatermark(@Param("watermarkId") long watermarkId); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java index 70487c313..91b512eac 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileExchangeImportExportTest.java @@ -123,7 +123,7 @@ public class FileExchangeImportExportTest extends AbstractPersistenceServerServi private FileExchangeExportRequest buildDefaultRequest() { - return new FileExchangeExportRequest(Collections.emptyList(), Collections.emptyList(), false, false, false, false, true); + return new FileExchangeExportRequest(Collections.emptySet(), Collections.emptySet(), false, false, false, false, true); } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java index c5af4f4bc..d4e4ff97e 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java @@ -1,13 +1,14 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.util.List; +import java.util.Set; import io.swagger.v3.oas.annotations.media.Schema; public record FileExchangeExportRequest( - @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") List dossierIds, - @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") List fileIds, - @Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS and NER_ENTITIES file will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles, + @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") Set dossierIds, + @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") Set fileIds, + @Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles, @Schema(description = "If set to true, the MANUAL_REDACTIONS file will be excluded from the export.", defaultValue = "false") boolean excludeManualRedactions, @Schema(description = "If set to true, the ENTITY_LOG and COMPONENT_LOG (with its overrides) will be excluded from the export.", defaultValue = "false") boolean excludeAnalysisLogs, @Schema(description = "If set to true, the File Attributes will be excluded from the export.", defaultValue = "false") boolean excludeFileAttributes, diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java index 60768e994..7c2d24589 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileStatusFilter.java @@ -2,7 +2,9 @@ package com.iqser.red.service.persistence.service.v1.api.shared.model; import java.util.ArrayList; import java.util.List; +import java.util.function.Predicate; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -29,4 +31,21 @@ public class FileStatusFilter { this.includeHardDeletedFiles = false; } + + public Predicate asPredicate() { + + if (this.getProcessingStatusList() == null) { + this.setProcessingStatusList(new ArrayList<>()); + } + + if (this.getWorkflowStatusList() == null) { + this.setWorkflowStatusList(new ArrayList<>()); + } + + return fileStatus -> (this.getProcessingStatusList().isEmpty() || this.getProcessingStatusList().contains(fileStatus.getProcessingStatus())) + && (this.getWorkflowStatusList().isEmpty() || this.getWorkflowStatusList().contains(fileStatus.getWorkflowStatus())) + && (this.isIncludeSoftDeletedFiles() || fileStatus.getDeleted() == null) + && (this.isIncludeHardDeletedFiles() || fileStatus.getHardDeletedTime() == null); + } + } diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java new file mode 100644 index 000000000..da05515b3 --- /dev/null +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/ReanalysisSettings.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.persistence.service.v1.api.shared.model; + +import java.util.Optional; +import java.util.Set; + +import io.swagger.v3.oas.annotations.media.Schema; + +public record ReanalysisSettings( + @Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for reanalysis.", defaultValue = "[]") Set dossierIds, + @Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for reanalysis.", defaultValue = "[]") Set fileIds, + @Schema(description = "If set to true, layout parsing and named entity recognition will be repeated.", defaultValue = "false") boolean repeatStructureAnalysis, + @Schema(description = "Use this to create a filter for files to reanalyse. Matches anything if set to null.", defaultValue = "{}") FileStatusFilter fileStatusFilter +) { + + public FileStatusFilter fileStatusFilter() { + + return Optional.ofNullable(fileStatusFilter) + .orElse(new FileStatusFilter()); + } + +}