From 835e3568f6338faa323e20e4af659f7d09cf3158 Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Tue, 23 Jan 2024 12:56:34 +0100 Subject: [PATCH] RED-7376: Flag in deployment to indicate supported file formats --- .../impl/controller/LicenseController.java | 99 ++-------------- .../api/impl/controller/UploadController.java | 32 +++-- .../service/FileFormatValidationService.java | 93 +++++++++++++++ .../service/LicenseUtilityService.java | 111 ++++++++++++++++++ 4 files changed, 240 insertions(+), 95 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileFormatValidationService.java create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/LicenseUtilityService.java diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/LicenseController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/LicenseController.java index b284cc556..b2339e659 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/LicenseController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/LicenseController.java @@ -2,19 +2,14 @@ package com.iqser.red.persistence.service.v1.external.api.impl.controller; import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_LICENSE; import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.UPDATE_LICENSE; +import static com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService.*; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.core.env.Environment; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.RestController; +import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService; +import com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService; import com.iqser.red.service.persistence.service.v1.api.external.resource.LicenseResource; -import com.iqser.red.service.persistence.service.v1.api.shared.model.license.Feature; -import com.iqser.red.service.persistence.service.v1.api.shared.model.license.FeatureType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.license.License; import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -28,18 +23,9 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class LicenseController implements LicenseResource { - private static final String LICENSE_OBJECT_ID = "license/redact-manager-license.json"; - private static final String DEMO_PDFTRON_LICENSE = "demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a"; - private static final String LICENSE_CUSTOMER = "LICENSE_CUSTOMER"; - private static final String LICENSE_START = "LICENSE_START"; - private static final String LICENSE_END = "LICENSE_END"; - private static final String LICENSE_PAGE_COUNT = "LICENSE_PAGE_COUNT"; - private static final String PDFTRON_FRONTEND_LICENSE = "PDFTRON_FRONTEND_LICENSE"; - private static final String PDFTRON_FEATURE = "pdftron"; - private static final String PROCESSING_PAGES_FEATURE = "processingPages"; - private static final String DEFAULT_PROCESSING_PAGES = "200000"; private final StorageService storageService; - private final Environment environment; + private final LicenseUtilityService licenseUtilityService; + private final FileFormatValidationService fileFormatValidationService; @Override @@ -48,6 +34,7 @@ public class LicenseController implements LicenseResource { public void updateLicense(RedactionLicenseModel licenseModel) { storageService.storeJSONObject(TenantContext.getTenantId(), LICENSE_OBJECT_ID, licenseModel); + fileFormatValidationService.evictValidFileFormatsForTenant(TenantContext.getTenantId()); } @@ -55,80 +42,16 @@ public class LicenseController implements LicenseResource { @PreAuthorize("hasAuthority('" + READ_LICENSE + "')") public RedactionLicenseModel getLicense() { - if (storageService.objectExists(TenantContext.getTenantId(), LICENSE_OBJECT_ID)) { + String tenantId = TenantContext.getTenantId(); + if (storageService.objectExists(tenantId, LICENSE_OBJECT_ID)) { try { - return storageService.readJSONObject(TenantContext.getTenantId(), LICENSE_OBJECT_ID, RedactionLicenseModel.class); + return storageService.readJSONObject(tenantId, LICENSE_OBJECT_ID, RedactionLicenseModel.class); } catch (Exception e) { - return generateDemoLicense(); + return licenseUtilityService.generateDemoLicense(); } } else { - return generateDemoLicense(); + return licenseUtilityService.generateDemoLicense(); } } - - private RedactionLicenseModel generateDemoLicense() { - - License demo = new License(); - demo.setId("RedactManager"); - demo.setName("RedactManager License"); - demo.setProduct("RedactManager"); - - var licensedTo = environment.getProperty(LICENSE_CUSTOMER, "RedactManager Demo License"); - demo.setLicensedTo(licensedTo); - - var licenseStartDate = parseDate(environment.getProperty(LICENSE_START)); - var start = licenseStartDate != null ? licenseStartDate : OffsetDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).truncatedTo(ChronoUnit.SECONDS); - demo.setValidFrom(start); - - var licenseEndDate = parseDate(environment.getProperty(LICENSE_END)); - var end = licenseEndDate != null ? licenseEndDate : OffsetDateTime.now() - .withMonth(12) - .withDayOfMonth(31) - .withHour(0) - .withMinute(0) - .withSecond(0) - .truncatedTo(ChronoUnit.SECONDS); - demo.setValidUntil(end); - - var pdftronLicense = environment.getProperty(PDFTRON_FRONTEND_LICENSE, DEMO_PDFTRON_LICENSE); - demo.getFeatures().add(Feature.builder().name(PDFTRON_FEATURE).type(FeatureType.STRING).value(pdftronLicense).build()); - - var pageCount = Long.parseLong(environment.getProperty(LICENSE_PAGE_COUNT, DEFAULT_PROCESSING_PAGES)); - demo.getFeatures().add(Feature.builder().name(PROCESSING_PAGES_FEATURE).type(FeatureType.NUMBER).value(pageCount).build()); - - var demoLicense = new RedactionLicenseModel(); - demoLicense.setActiveLicense("RedactManager"); - demoLicense.getLicenses().add(demo); - - return demoLicense; - } - - - private OffsetDateTime parseDate(String date) { - - if (StringUtils.isEmpty(date)) { - return null; - } - var parts = date.split("-"); - if (parts.length != 3) { - return null; - } - - try { - return OffsetDateTime.now() - .withYear(Integer.parseInt(parts[2])) - .withMonth(Integer.parseInt(parts[1])) - .withDayOfMonth(Integer.parseInt(parts[0])) - .withHour(0) - .withMinute(0) - .withSecond(0) - .truncatedTo(ChronoUnit.SECONDS); - } catch (Exception e) { - log.debug("Failed to parse date: {}", date); - return null; - } - - } - } diff --git a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java index b0849b3fc..5e7c22d28 100644 --- a/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java +++ b/persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/UploadController.java @@ -7,7 +7,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -22,6 +21,8 @@ import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException; import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity; import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; @@ -34,6 +35,7 @@ import com.iqser.red.service.persistence.service.v1.api.external.resource.Upload import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest; +import com.knecon.fforesight.tenantcommons.TenantContext; import feign.FeignException; import io.micrometer.core.annotation.Timed; @@ -52,12 +54,11 @@ public class UploadController implements UploadResource { private static final int THRESHOLD_SIZE = 1000000000; // 1 GB private static final double THRESHOLD_RATIO = 10; - private static final List VALID_FILE_EXTENSIONS = List.of("pdf", "docx", "doc", "xls", "xlsx", "ppt", "pptx"); - private final UploadService uploadService; private final ReanalysisService reanalysisService; private final AccessControlService accessControlService; private final AuditPersistenceService auditPersistenceService; + private final FileFormatValidationService fileFormatValidationService; @Timed @@ -79,11 +80,13 @@ public class UploadController implements UploadResource { case "csv": return uploadService.importCsv(dossierId, file.getBytes()); default: - if (VALID_FILE_EXTENSIONS.contains(extension)) { - return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions); - } else { + if (!fileFormatValidationService.getAllFileFormats().contains(extension)) { throw new BadRequestException("Invalid file uploaded"); } + if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) { + throw new NotAllowedException("Insufficient permissions"); + } + return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions); } } catch (IOException e) { throw new BadRequestException(e.getMessage(), e); @@ -146,6 +149,12 @@ public class UploadController implements UploadResource { log.debug("CSV file inside ZIP failed", e); // TODO return un-processed files to client } + } else if (zipData.fileUploadResult.getFileIds().isEmpty()) { + if (zipData.containedUnpermittedFiles) { + throw new NotAllowedException("Zip file contains unpermitted files"); + } else { + throw new BadRequestException("Only unsupported files in zip file"); + } } return zipData.fileUploadResult; @@ -194,6 +203,7 @@ public class UploadController implements UploadResource { private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData) throws IOException { var extension = getExtension(ze.getName()); + final String fileName; if (ze.getName().lastIndexOf("/") >= 0) { fileName = ze.getName().substring(ze.getName().lastIndexOf("/") + 1); @@ -216,7 +226,14 @@ public class UploadController implements UploadResource { if ("csv".equals(extension)) { zipData.csvBytes = entryAsBytes; - } else if (VALID_FILE_EXTENSIONS.contains(extension)) { + } else if (fileFormatValidationService.getAllFileFormats().contains(extension)) { + + if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) { + zipData.containedUnpermittedFiles = true; + return; + } + zipData.containedUnpermittedFiles = false; + try { var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions); zipData.fileUploadResult.getFileIds().addAll(result.getFileIds()); @@ -260,6 +277,7 @@ public class UploadController implements UploadResource { int totalSizeArchive; int totalEntryArchive; FileUploadResult fileUploadResult = new FileUploadResult(); + boolean containedUnpermittedFiles; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileFormatValidationService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileFormatValidationService.java new file mode 100644 index 000000000..4a84997c3 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/FileFormatValidationService.java @@ -0,0 +1,93 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import static com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService.*; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.license.Feature; +import com.iqser.red.service.persistence.service.v1.api.shared.model.license.License; +import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel; +import com.iqser.red.storage.commons.service.StorageService; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class FileFormatValidationService { + + final StorageService storageService; + + final LicenseUtilityService licenseUtilityService; + + static Map> validFileFormatsCache = new HashMap<>(); + + + public List getAllFileFormats() { + + return ALL_FILE_FORMATS; + } + + + public List getValidFileFormatsForTenant(String tenantId) { + + boolean supportMSOfficeFormats = false; + + if (validFileFormatsCache.containsKey(tenantId)) { + return validFileFormatsCache.get(tenantId); + + } else { + Optional optionalLicense = getActiveLicense(tenantId); + if (optionalLicense.isPresent()) { + Optional optionalSupportMSOfficeFormatsFeature = optionalLicense.get() + .getFeatures() + .stream() + .filter(f -> f.getName().equals(SUPPORT_MS_OFFICE_FORMATS_FEATURE)) + .findFirst(); + + supportMSOfficeFormats = optionalSupportMSOfficeFormatsFeature.isPresent() ? (Boolean) optionalSupportMSOfficeFormatsFeature.get().getValue() : false; + } + List validFileFormats = supportMSOfficeFormats ? ALL_FILE_FORMATS : FILE_FORMATS_EXCLUDING_MS_OFFICE; + + validFileFormatsCache.put(tenantId, validFileFormats); + return validFileFormats; + } + } + + + public void evictValidFileFormatsForTenant(String tenantId) { + + validFileFormatsCache.remove(tenantId); + + } + + + private Optional getActiveLicense(String tenantId) { + + var redactionLicenseModel = getLicenseForTenant(tenantId); + return redactionLicenseModel.getLicenses().stream().filter(rlm -> Objects.equals(rlm.getId(), redactionLicenseModel.getActiveLicense())).findFirst(); + + } + + + private RedactionLicenseModel getLicenseForTenant(String tenantId) { + + if (storageService.objectExists(tenantId, LICENSE_OBJECT_ID)) { + try { + return storageService.readJSONObject(tenantId, LICENSE_OBJECT_ID, RedactionLicenseModel.class); + } catch (Exception e) { + return licenseUtilityService.generateDemoLicense(); + } + } + return licenseUtilityService.generateDemoLicense(); + } + +} \ No newline at end of file diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/LicenseUtilityService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/LicenseUtilityService.java new file mode 100644 index 000000000..db510480f --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/LicenseUtilityService.java @@ -0,0 +1,111 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.license.Feature; +import com.iqser.red.service.persistence.service.v1.api.shared.model.license.FeatureType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.license.License; +import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@AllArgsConstructor +public class LicenseUtilityService { + + public static final String LICENSE_OBJECT_ID = "license/redact-manager-license.json"; + public static final String DEMO_PDFTRON_LICENSE = "demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a"; + public static final String LICENSE_CUSTOMER = "LICENSE_CUSTOMER"; + public static final String LICENSE_START = "LICENSE_START"; + public static final String LICENSE_END = "LICENSE_END"; + public static final String LICENSE_PAGE_COUNT = "LICENSE_PAGE_COUNT"; + public static final String LICENSE_MS_OFFICE_FORMATS = "LICENSE_SUPPORT_MS_OFFICE_FORMATS"; + public static final String PDFTRON_FRONTEND_LICENSE = "PDFTRON_FRONTEND_LICENSE"; + public static final String PDFTRON_FEATURE = "pdftron"; + public static final String PROCESSING_PAGES_FEATURE = "processingPages"; + public static final String DEFAULT_PROCESSING_PAGES = "200000"; + public static final String SUPPORT_MS_OFFICE_FORMATS_FEATURE = "supportMSOfficeFormats"; + public static final String DEFAULT_SUPPORT_MS_OFFICE_FORMATS = "false"; + + public static final List ALL_FILE_FORMATS = List.of("pdf", "docx", "doc", "xls", "xlsx", "ppt", "pptx"); + public static final List FILE_FORMATS_EXCLUDING_MS_OFFICE = List.of("pdf"); + + private final Environment environment; + + + public RedactionLicenseModel generateDemoLicense() { + + License demo = new License(); + demo.setId("RedactManager"); + demo.setName("RedactManager License"); + demo.setProduct("RedactManager"); + + var licensedTo = environment.getProperty(LICENSE_CUSTOMER, "RedactManager Demo License"); + demo.setLicensedTo(licensedTo); + + var licenseStartDate = parseDate(environment.getProperty(LICENSE_START)); + var start = licenseStartDate != null ? licenseStartDate : OffsetDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).truncatedTo(ChronoUnit.SECONDS); + demo.setValidFrom(start); + + var licenseEndDate = parseDate(environment.getProperty(LICENSE_END)); + var end = licenseEndDate != null ? licenseEndDate : OffsetDateTime.now() + .withMonth(12) + .withDayOfMonth(31) + .withHour(0) + .withMinute(0) + .withSecond(0) + .truncatedTo(ChronoUnit.SECONDS); + demo.setValidUntil(end); + + var pdftronLicense = environment.getProperty(PDFTRON_FRONTEND_LICENSE, DEMO_PDFTRON_LICENSE); + demo.getFeatures().add(Feature.builder().name(PDFTRON_FEATURE).type(FeatureType.STRING).value(pdftronLicense).build()); + + var pageCount = Long.parseLong(environment.getProperty(LICENSE_PAGE_COUNT, DEFAULT_PROCESSING_PAGES)); + demo.getFeatures().add(Feature.builder().name(PROCESSING_PAGES_FEATURE).type(FeatureType.NUMBER).value(pageCount).build()); + + boolean supportMSOfficeFormats = Boolean.parseBoolean(environment.getProperty(LICENSE_MS_OFFICE_FORMATS, DEFAULT_SUPPORT_MS_OFFICE_FORMATS)); + demo.getFeatures().add(Feature.builder().name(SUPPORT_MS_OFFICE_FORMATS_FEATURE).type(FeatureType.BOOLEAN).value(supportMSOfficeFormats).build()); + + var demoLicense = new RedactionLicenseModel(); + demoLicense.setActiveLicense("RedactManager"); + demoLicense.getLicenses().add(demo); + + return demoLicense; + } + + + private OffsetDateTime parseDate(String date) { + + if (StringUtils.isEmpty(date)) { + return null; + } + var parts = date.split("-"); + if (parts.length != 3) { + return null; + } + + try { + return OffsetDateTime.now() + .withYear(Integer.parseInt(parts[2])) + .withMonth(Integer.parseInt(parts[1])) + .withDayOfMonth(Integer.parseInt(parts[0])) + .withHour(0) + .withMinute(0) + .withSecond(0) + .truncatedTo(ChronoUnit.SECONDS); + } catch (Exception e) { + log.debug("Failed to parse date: {}", date); + return null; + } + + } + +}