Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-9255: implement file exchange

Browse Source 
* add simplified text file to layoutparsing file export
* add validation for fileId/dossierId combinations
This commit is contained in:
Kilian Schuettler 2024-06-20 11:40:52 +02:00
parent cdece93037
commit 46f9523b2c
7 changed files with 95 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/SupportController.java vendored
View File

@ -50,7 +50,6 @@ public class SupportController implements SupportResource {
@Override
public void reanalyzeFiles(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
log.info("Scheduling reanalysis for all files in Dossier Tempalte {} with settings {}", dossierTemplateId, reanalysisSettings);
reanalysisService.reanalyzeTemplate(dossierTemplateId, reanalysisSettings);
}

13
persistence-service-v1/persistence-service-external-api-impl-v2/src/main/java/com/iqser/red/persistence/service/v2/external/api/impl/controller/DossierTemplateControllerV2.java vendored
View File

@ -56,6 +56,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.Audit
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionAddRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinitionUpdateRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.DroolsValidationResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.rules.RulesUploadRequest;
import com.iqser.red.service.persistence.service.v2.api.external.model.ComponentMappingMetadataModel;
@ -216,12 +217,12 @@ public class DossierTemplateControllerV2 implements DossierTemplateResource {
Path mappingFile = saveToFile(file);
String fileName = file.getOriginalFilename() == null ? nameToUse + ".csv" : file.getOriginalFilename();
com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
nameToUse,
fileName,
delimiter,
encoding,
mappingFile.toFile());
ComponentMappingMetadata metaData = componentMappingService.create(dossierTemplateId,
nameToUse,
fileName,
delimiter,
encoding,
mappingFile.toFile());
Files.deleteIfExists(mappingFile);

19
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/SupportResource.java vendored
View File

@ -51,8 +51,23 @@ public interface SupportResource {
String IMPORT = "/import";
@PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
@Operation(summary = "Reanalyze all files in dossier template", description = "None")
@PostMapping(value = REANALYSIS_REST_PATH + DOSSIER_TEMPLATE_ID_PATH_VARIABLE)
@Operation(summary = "Reanalyze all files in dossier template", description = """
## Reanalyze Files Endpoint
Use this endpoint to reanalyze all files in a specified Dossier Template. The reanalysis process can be tailored using various filtering options provided in the request body.
### Parameters
- **DossierTemplateId**: Specifies the Dossier Template whose files need to be reanalyzed.
### Request Body Configuration Options
- **dossierIds**: List of dossier IDs to filter. If empty, all dossiers are selected for reanalysis.
- **fileIds**: List of file IDs to filter. If empty, all files are selected for reanalysis.
- **repeatStructureAnalysis**: Boolean. If true, layout parsing and named entity recognition will be repeated.
- **fileStatusFilter**: Use this to create a filter for files to reanalyze. Matches any file if set to null.
""")
@ApiResponses(value = {@ApiResponse(responseCode = "204", description = "OK"), @ApiResponse(responseCode = "403", description = "Forbidden")})
void reanalyzeFiles(@PathVariable(DOSSIER_TEMPLATE_ID) String dossierTemplateId,
@RequestBody ReanalysisSettings reAnalysisSettings);

41
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/dataexchange/service/FileExchangeExportService.java
View File

@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.management.v1.processor.dataexchange.Ex
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.FileExchangeNames;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierIdFileIdRequestValidator;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusManagementService;
@ -61,6 +62,8 @@ public class FileExchangeExportService {
ObjectMapper mapper;
RabbitTemplate rabbitTemplate;
EntityTypeExportService entityTypeExportService;
DossierIdFileIdRequestValidator requestValidator;
@Observed(name = "FileExchangeExportService", contextualName = "prepare-export")
public DownloadResponse prepareExportDownload(String dossierTemplateId, FileExchangeExportRequest request) {
@ -69,7 +72,7 @@ public class FileExchangeExportService {
DossierTemplateEntity dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId);
String dossierTemplateName = dossierTemplate.getName();
validateRequest(dossierTemplate, request);
requestValidator.validateRequestOrThrow404(dossierTemplateId, request.dossierIds(), request.fileIds());
String downloadFilename = dossierTemplateName + "_file-exchange.zip";
String storageId = StorageIdUtils.getStorageId(KeycloakSecurity.getUserId(), dossierTemplateId + "_file-exchange");
@ -87,42 +90,6 @@ public class FileExchangeExportService {
return new DownloadResponse(storageId);
}
@Observed(name = "FileExchangeExportService", contextualName = "validate-request")
private void validateRequest(DossierTemplateEntity dossierTemplate, FileExchangeExportRequest request) {
Set<String> dossierIds = new HashSet<>(request.dossierIds());
if (!request.dossierIds().isEmpty()) {
Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplate.getId());
Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
if (!nonAvailableDossiers.isEmpty()) {
throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s",
String.join(", ", nonAvailableDossiers),
dossierTemplate.getId()));
}
}
if (!request.fileIds().isEmpty()) {
Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplate.getId())
.stream()
.filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
.map(FileModel::getId)
.collect(Collectors.toSet());
Set<String> nonAvailableFiles = Sets.difference(new HashSet<>(request.fileIds()), availableFileIds);
if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) {
throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplate.getId()));
}
if (!nonAvailableFiles.isEmpty()) {
throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s",
String.join(", ", nonAvailableFiles),
String.join(", ", dossierIds),
dossierTemplate.getId()));
}
}
}
private void addToExportDownloadQueue(ExportDownloadMessage downloadJob) {

59
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierIdFileIdRequestValidator.java Normal file
View File

@ -0,0 +1,59 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class DossierIdFileIdRequestValidator {
DossierManagementService dossierManagementService;
FileStatusManagementService fileStatusManagementService;
@Observed(name = "DossierIdFileIdRequestValidator", contextualName = "validate-request")
public void validateRequestOrThrow404(String dossierTemplateId, Set<String> dossierIds, Set<String> fileIds) {
if (!dossierIds.isEmpty()) {
Set<String> availableDossierIds = dossierManagementService.getAllDossierIdsForDossierTemplateId(dossierTemplateId);
Set<String> nonAvailableDossiers = Sets.difference(dossierIds, availableDossierIds);
if (!nonAvailableDossiers.isEmpty()) {
throw new NotFoundException(String.format("Dossier Ids %s are not available in dossier template %s", String.join(", ", nonAvailableDossiers), dossierTemplateId));
}
}
if (!fileIds.isEmpty()) {
Set<String> availableFileIds = fileStatusManagementService.getAllDossierTemplateStatus(dossierTemplateId)
.stream()
.filter(fileModel -> dossierIds.isEmpty() || dossierIds.contains(fileModel.getDossierId()))
.map(FileModel::getId)
.collect(Collectors.toSet());
Set<String> nonAvailableFiles = Sets.difference(fileIds, availableFileIds);
if (!nonAvailableFiles.isEmpty() && dossierIds.isEmpty()) {
throw new NotFoundException(String.format("File Ids %s are not found in dossier template %s", String.join(", ", nonAvailableFiles), dossierTemplateId));
}
if (!nonAvailableFiles.isEmpty()) {
throw new NotFoundException(String.format("File Ids %s are not found in any of the dossiers %s in dossier template %s",
String.join(", ", nonAvailableFiles),
String.join(", ", dossierIds),
dossierTemplateId));
}
}
}
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/ReanalysisService.java
View File

@ -5,11 +5,13 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
import com.iqser.red.service.persistence.management.v1.processor.client.pdftronredactionservice.PDFTronClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.external.model.ReanalysisSettings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileExchangeExportRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.DeleteImportedRedactionsRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -26,6 +28,7 @@ import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
@ -41,6 +44,7 @@ public class ReanalysisService {
private final IndexingService indexingService;
private final PDFTronClient pDFTronRedactionClient;
private final FileManagementStorageService fileManagementStorageService;
private final DossierIdFileIdRequestValidator requestValidator;
private final Predicate<FileModel> validFilesFilter = fileStatus -> !fileStatus.isSoftOrHardDeleted() && !fileStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED);
private final Predicate<FileModel> errorFilesFilter = fileStatus -> fileStatus.getProcessingStatus().equals(ProcessingStatus.ERROR);
@ -268,6 +272,7 @@ public class ReanalysisService {
public void reanalyzeTemplate(String dossierTemplateId, ReanalysisSettings reanalysisSettings) {
requestValidator.validateRequestOrThrow404(dossierTemplateId, reanalysisSettings.dossierIds(), reanalysisSettings.fileIds());
fileStatusService.getDossierTemplateStatus(dossierTemplateId)
.stream()
.filter(file -> isInList(file, reanalysisSettings))

5
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/FileExchangeExportRequest.java
View File

@ -1,12 +1,13 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model;
import java.util.List;
import java.util.Set;
import io.swagger.v3.oas.annotations.media.Schema;
public record FileExchangeExportRequest(
@Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") List<String> dossierIds,
@Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") List<String> fileIds,
@Schema(description = "Provide a list of dossierIds to filter for. If the list is empty, every dossier is selected for export.", defaultValue = "[]") Set<String> dossierIds,
@Schema(description = "Provide a list of fileIds to filter for. If the list is empty, every file is selected for export.", defaultValue = "[]") Set<String> fileIds,
@Schema(description = "If set to true, the DOCUMENT_STRUCTURE/_PAGES/_TEXT/_POSITIONS, SIMPLIFIED_TEXT, and NER_ENTITIES files will be excluded from the export.", defaultValue = "false") boolean excludeLayoutFiles,
@Schema(description = "If set to true, the MANUAL_REDACTIONS file will be excluded from the export.", defaultValue = "false") boolean excludeManualRedactions,
@Schema(description = "If set to true, the ENTITY_LOG and COMPONENT_LOG (with its overrides) will be excluded from the export.", defaultValue = "false") boolean excludeAnalysisLogs,