From 4fb19f14a98fba1013fd8de98cf4e707c8af359d Mon Sep 17 00:00:00 2001 From: Ali Oezyetimoglu Date: Tue, 21 Jan 2025 20:22:25 +0100 Subject: [PATCH] RED-10730: added internal endpoints for dossier-initialization-service --- .../controller/DossierInternalController.java | 13 +- .../DossierTemplateInternalController.java | 9 + .../controller/UploadInternalController.java | 34 +++ .../internal/resources/DossierResource.java | 9 + .../resources/DossierTemplateResource.java | 6 + .../internal/resources/UploadResource.java | 33 +++ .../service/DossierManagementService.java | 53 ++-- .../service/UploadManagementService.java | 254 ++++++++++++++++++ 8 files changed, 391 insertions(+), 20 deletions(-) create mode 100644 persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/UploadInternalController.java create mode 100644 persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/UploadResource.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadManagementService.java diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierInternalController.java index 686b4eae8..47ac2119f 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierInternalController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierInternalController.java @@ -2,17 +2,18 @@ package com.iqser.red.service.persistence.v1.internal.api.controller; import java.util.List; -import jakarta.transaction.Transactional; - import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.persistence.management.v1.processor.service.DossierManagementService; import com.iqser.red.service.persistence.service.v1.api.internal.resources.DossierResource; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; import feign.Param; +import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -42,4 +43,12 @@ public class DossierInternalController implements DossierResource { return dossierManagementService.getDossierById(dossierId, includeArchived, includeDeleted); } + + @Override + @Transactional + public Dossier createDossier(@RequestBody CreateOrUpdateDossierRequest dossierRequest) { + + return dossierManagementService.createDossier(dossierRequest); + } + } diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierTemplateInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierTemplateInternalController.java index 5b70ec397..617174c43 100644 --- a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierTemplateInternalController.java +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DossierTemplateInternalController.java @@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.v1.internal.api.controller; import static com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter.convert; +import java.util.List; + import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; @@ -37,4 +39,11 @@ public class DossierTemplateInternalController implements DossierTemplateResourc return convert(dossierTemplatePersistenceService.getDossierTemplate(dossierTemplateId), DossierTemplate.class); } + + @Override + public List getAllDossierTemplates() { + + return convert(dossierTemplatePersistenceService.getAllDossierTemplates(), DossierTemplate.class); + } + } diff --git a/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/UploadInternalController.java b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/UploadInternalController.java new file mode 100644 index 000000000..1a23581cb --- /dev/null +++ b/persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/UploadInternalController.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.persistence.v1.internal.api.controller; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import com.iqser.red.service.persistence.management.v1.processor.service.UploadManagementService; +import com.iqser.red.service.persistence.service.v1.api.internal.resources.UploadResource; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult; + +import io.swagger.v3.oas.annotations.Parameter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@RestController +@RequiredArgsConstructor +@Slf4j +public class UploadInternalController implements UploadResource { + + private final UploadManagementService uploadManagementService; + + + @Override + public FileUploadResult upload(@RequestPart(name = "file") MultipartFile file, + @PathVariable(DOSSIER_ID) String dossierId, + @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions, + @Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic redaction for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis) { + + return uploadManagementService.upload(file, dossierId, keepManualRedactions, disableAutomaticAnalysis); + } + +} diff --git a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierResource.java b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierResource.java index 8ae8e04ad..78e669476 100644 --- a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierResource.java +++ b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierResource.java @@ -6,9 +6,13 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier; @ResponseStatus(value = HttpStatus.OK) @@ -33,4 +37,9 @@ public interface DossierResource { @RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived, @RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted); + + @ResponseBody + @PostMapping(value = InternalApi.BASE_PATH + REST_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + Dossier createDossier(@RequestBody CreateOrUpdateDossierRequest dossierRequest); + } diff --git a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierTemplateResource.java b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierTemplateResource.java index 4a95d2b80..617ea93f9 100644 --- a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierTemplateResource.java +++ b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/DossierTemplateResource.java @@ -1,5 +1,7 @@ package com.iqser.red.service.persistence.service.v1.api.internal.resources; +import java.util.List; + import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -29,4 +31,8 @@ public interface DossierTemplateResource { @GetMapping(value = InternalApi.BASE_PATH + DOSSIER_TEMPLATE_PATH + DOSSIER_ID_PATH_PARAM, produces = MediaType.APPLICATION_JSON_VALUE) DossierTemplate getDossierTemplateById(@PathVariable(DOSSIER_TEMPLATE_ID_PARAM) String dossierTemplateId); + + @GetMapping(value = InternalApi.BASE_PATH + DOSSIER_TEMPLATE_PATH, produces = MediaType.APPLICATION_JSON_VALUE) + List getAllDossierTemplates(); + } diff --git a/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/UploadResource.java b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/UploadResource.java new file mode 100644 index 000000000..b2a7c5e27 --- /dev/null +++ b/persistence-service-v1/persistence-service-internal-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/internal/resources/UploadResource.java @@ -0,0 +1,33 @@ +package com.iqser.red.service.persistence.service.v1.api.internal.resources; + +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.multipart.MultipartFile; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult; + +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; + +public interface UploadResource { + + String DOSSIER_ID = "dossierId"; + String FILE_ID = "fileId"; + String UPLOAD_PATH = InternalApi.BASE_PATH + "/upload"; + String DOSSIER_ID_PATH_VARIABLE = "/{" + DOSSIER_ID + "}"; + String FILE_ID_PATH_VARIABLE = "/{" + FILE_ID + "}"; + String DISABLE_AUTOMATIC_ANALYSIS_PARAM = "disableAutomaticAnalysis"; + + + @ResponseBody + @PostMapping(value = UPLOAD_PATH + DOSSIER_ID_PATH_VARIABLE, consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + FileUploadResult upload(@Schema(type = "string", format = "binary", name = "file") @RequestPart(name = "file") MultipartFile file, + @PathVariable(DOSSIER_ID) String dossierId, + @RequestParam(value = "keepManualRedactions", required = false, defaultValue = "false") boolean keepManualRedactions, + @Parameter(name = DISABLE_AUTOMATIC_ANALYSIS_PARAM, description = "Disables automatic analysis for the uploaded file, imports only imported redactions") @RequestParam(value = DISABLE_AUTOMATIC_ANALYSIS_PARAM, required = false, defaultValue = "false") boolean disableAutomaticAnalysis); + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java index cd2bd6279..f0b5b6f9c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierManagementService.java @@ -1,5 +1,16 @@ package com.iqser.red.service.persistence.management.v1.processor.service; +import static com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException.DOSSIER_NOT_FOUND_MESSAGE; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException; import com.iqser.red.service.persistence.management.v1.processor.utils.DossierMapper; @@ -17,17 +28,6 @@ import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException.DOSSIER_NOT_FOUND_MESSAGE; - @Slf4j @Service @RequiredArgsConstructor @@ -47,6 +47,13 @@ public class DossierManagementService { } + @Transactional + public Dossier createDossier(CreateOrUpdateDossierRequest dossierRequest) { + + return MagicConverter.convert(dossierService.addDossier(dossierRequest), Dossier.class, new DossierMapper()); + } + + @Transactional public Dossier updateDossier(CreateOrUpdateDossierRequest dossierRequest, String dossierId) { @@ -65,8 +72,9 @@ public class DossierManagementService { List fileStatuses = fileStatusService.getDossierStatus(dossierId); var relevantFileIds = fileStatuses.stream() - .filter(fileStatus -> fileStatus.getDeleted() == null).map(FileModel::getId).toList(); - + .filter(fileStatus -> fileStatus.getDeleted() == null) + .map(FileModel::getId) + .toList(); dossierDeletionService.softDeleteDossier(dossierId, relevantFileIds, now); fileDeletionService.reindexDeletedFiles(dossierId, relevantFileIds); @@ -120,12 +128,14 @@ public class DossierManagementService { } + @Transactional public List getAllDossierIdsForDossierTemplateId(String dossierTemplateId, boolean includeArchived, boolean includeDeleted) { return dossierService.getAllDossierIdsForDossierTemplateId(dossierTemplateId, includeArchived, includeDeleted); } + @Transactional public List findAllDossierIdsInDossierTemplateId(String dossierTemplateId, Set dossierIds) { @@ -215,7 +225,10 @@ public class DossierManagementService { for (String dossierId : dossierIds) { - List fileIds = fileStatusService.getDossierStatus(dossierId).stream().map(FileModel::getId).collect(Collectors.toList()); + List fileIds = fileStatusService.getDossierStatus(dossierId) + .stream() + .map(FileModel::getId) + .collect(Collectors.toList()); dossierDeletionService.hardDeleteDossier(dossierId, fileIds); dossierDeletionService.hardDeleteFileDataAndIndexUpdates(dossierId, fileIds); } @@ -228,10 +241,13 @@ public class DossierManagementService { for (String dossierId : dossierIds) { var dossier = dossierService.getDossierById(dossierId); List fileStatuses = fileStatusService.getDossierStatus(dossierId); - var relevantFileIds = fileStatuses.stream().filter(fileStatus -> fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted() - .isAfter(dossier.getSoftDeletedTime()))).map(FileModel::getId).collect(Collectors.toList()); + var relevantFileIds = fileStatuses.stream() + .filter(fileStatus -> fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted() + .isAfter(dossier.getSoftDeletedTime()))) + .map(FileModel::getId) + .collect(Collectors.toList()); - dossierDeletionService.undeleteDossier(dossierId,relevantFileIds,dossier.getSoftDeletedTime()); + dossierDeletionService.undeleteDossier(dossierId, relevantFileIds, dossier.getSoftDeletedTime()); dossierDeletionService.reindexUndeletedFiles(dossier.getDossierTemplateId(), dossierId, relevantFileIds); } @@ -278,6 +294,7 @@ public class DossierManagementService { public DossierChangeResponseV2 changesSinceV2(JSONPrimitive since) { + return dossierService.changesSinceV2(since.getValue()); } @@ -285,7 +302,7 @@ public class DossierManagementService { @Transactional public List getDossiersByIds(Set viewableDossierIds) { - return getConvertedAllDossiers(dossierService.getAllDossiers(viewableDossierIds), true,true); + return getConvertedAllDossiers(dossierService.getAllDossiers(viewableDossierIds), true, true); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadManagementService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadManagementService.java new file mode 100644 index 000000000..e5502f5a7 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/UploadManagementService.java @@ -0,0 +1,254 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Locale; +import java.util.UUID; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.io.IOUtils; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; +import com.iqser.red.service.persistence.management.v1.processor.utils.FileUtils; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult; +import com.knecon.fforesight.tenantcommons.TenantContext; + +import io.micrometer.core.annotation.Timed; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class UploadManagementService { + + private static final int THRESHOLD_ENTRIES = 10000; // Maximum number of files allowed + private static final int THRESHOLD_SIZE = 1000000000; // 1 GB total unzipped data + private static final double THRESHOLD_RATIO = 10; // Max allowed compression ratio + + private final UploadService uploadService; + private final AccessControlService accessControlService; + private final FileFormatValidationService fileFormatValidationService; + + + @Timed + public FileUploadResult upload(MultipartFile file, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) { + + String originalFilename = file.getOriginalFilename(); + if (originalFilename == null) { + throw new BadRequestException("Could not upload file, no filename provided."); + } + + String extension = getExtension(originalFilename); + try { + return switch (extension) { + case "zip" -> handleZip(dossierId, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis); + case "csv" -> uploadService.importCsv(dossierId, file.getBytes()); + default -> { + validateExtensionOrThrow(extension); + yield uploadService.processSingleFile(dossierId, originalFilename, file.getBytes(), keepManualRedactions, disableAutomaticAnalysis); + } + }; + } catch (IOException e) { + throw new BadRequestException("Failed to process file: " + e.getMessage(), e); + } + } + + + private void validateExtensionOrThrow(String extension) { + + if (!fileFormatValidationService.getAllFileFormats().contains(extension)) { + throw new BadRequestException("Invalid file uploaded (unrecognized extension)."); + } + if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) { + throw new NotAllowedException("Insufficient permissions for this file type."); + } + } + + + /** + * 1. Write the uploaded content to a temp ZIP file + * 2. Check the number of entries and reject if too big or if symlinks found + * 3. Unzip and process each file, while checking size and ratio. + */ + private FileUploadResult handleZip(String dossierId, byte[] fileContent, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException { + + File tempZip = FileUtils.createTempFile(UUID.randomUUID().toString(), ".zip"); + try (FileOutputStream fos = new FileOutputStream(tempZip)) { + IOUtils.write(fileContent, fos); + } + + validateZipEntries(tempZip); + + try { + ZipData zipData = processZipContents(tempZip, dossierId, keepManualRedactions, disableAutomaticAnalysis); + + if (zipData.csvBytes != null) { + try { + FileUploadResult csvResult = uploadService.importCsv(dossierId, zipData.csvBytes); + zipData.fileUploadResult.getProcessedAttributes().addAll(csvResult.getProcessedAttributes()); + zipData.fileUploadResult.getProcessedFileIds().addAll(csvResult.getProcessedFileIds()); + } catch (Exception e) { + log.debug("CSV file inside ZIP failed to import", e); + } + } else if (zipData.fileUploadResult.getFileIds().isEmpty()) { + if (zipData.containedUnpermittedFiles) { + throw new NotAllowedException("Zip file contains unpermitted files."); + } else { + throw new BadRequestException("Only unsupported files in the ZIP."); + } + } + + return zipData.fileUploadResult; + } finally { + + if (!tempZip.delete()) { + log.warn("Could not delete temporary ZIP file: {}", tempZip); + } + } + } + + + private void validateZipEntries(File tempZip) throws IOException { + + try (FileInputStream fis = new FileInputStream(tempZip); ZipFile zipFile = new ZipFile(fis.getChannel())) { + + int count = 0; + var entries = zipFile.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry ze = entries.nextElement(); + + if (ze.isUnixSymlink()) { + throw new BadRequestException("ZIP-files with symlinks are not allowed."); + } + + if (!ze.isDirectory() && !ze.getName().startsWith(".")) { + count++; + if (count > THRESHOLD_ENTRIES) { + throw new BadRequestException("ZIP-Bomb detected: too many entries."); + } + } + } + } + } + + + private ZipData processZipContents(File tempZip, String dossierId, boolean keepManualRedactions, boolean disableAutomaticAnalysis) throws IOException { + + ZipData zipData = new ZipData(); + + try (FileInputStream fis = new FileInputStream(tempZip); ZipFile zipFile = new ZipFile(fis.getChannel())) { + + var entries = zipFile.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + + if (entry.isDirectory() || entry.getName().startsWith(".")) { + continue; + } + + byte[] entryBytes = readEntryWithRatioCheck(entry, zipFile); + zipData.totalSizeArchive += entryBytes.length; + if (zipData.totalSizeArchive > THRESHOLD_SIZE) { + throw new BadRequestException("ZIP-Bomb detected (exceeds total size limit)."); + } + + String extension = getExtension(entry.getName()); + if ("csv".equalsIgnoreCase(extension)) { + zipData.csvBytes = entryBytes; + } else { + handleRegularFile(dossierId, entryBytes, extension, extractFileName(entry.getName()), zipData, keepManualRedactions, disableAutomaticAnalysis); + } + } + } + return zipData; + } + + + private byte[] readEntryWithRatioCheck(ZipArchiveEntry entry, ZipFile zipFile) throws IOException { + + long compressedSize = entry.getCompressedSize() > 0 ? entry.getCompressedSize() : 1; + try (var is = zipFile.getInputStream(entry); var bos = new ByteArrayOutputStream()) { + + byte[] buffer = new byte[4096]; + int bytesRead; + int totalUncompressed = 0; + + while ((bytesRead = is.read(buffer)) != -1) { + bos.write(buffer, 0, bytesRead); + totalUncompressed += bytesRead; + + double ratio = (double) totalUncompressed / compressedSize; + if (ratio > THRESHOLD_RATIO) { + throw new BadRequestException("ZIP-Bomb detected (compression ratio too high)."); + } + } + return bos.toByteArray(); + } + } + + + private void handleRegularFile(String dossierId, + byte[] fileBytes, + String extension, + String fileName, + ZipData zipData, + boolean keepManualRedactions, + boolean disableAutomaticAnalysis) { + + if (!fileFormatValidationService.getAllFileFormats().contains(extension)) { + zipData.containedUnpermittedFiles = false; + return; + } + + if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) { + zipData.containedUnpermittedFiles = true; + return; + } + + try { + FileUploadResult result = uploadService.processSingleFile(dossierId, fileName, fileBytes, keepManualRedactions, disableAutomaticAnalysis); + zipData.fileUploadResult.getFileIds().addAll(result.getFileIds()); + } catch (Exception e) { + log.debug("Failed to process file '{}' in ZIP: {}", fileName, e.getMessage(), e); + } + } + + + private String extractFileName(String path) { + + int idx = path.lastIndexOf('/'); + return (idx >= 0) ? path.substring(idx + 1) : path; + } + + + private String getExtension(String fileName) { + + int idx = fileName.lastIndexOf('.'); + if (idx < 0) { + return ""; + } + return fileName.substring(idx + 1).toLowerCase(Locale.ROOT); + } + + + @FieldDefaults(level = AccessLevel.PUBLIC) + private static final class ZipData { + + byte[] csvBytes; + int totalSizeArchive; + FileUploadResult fileUploadResult = new FileUploadResult(); + boolean containedUnpermittedFiles; + + } + +}