RED-7376: Flag in deployment to indicate supported file formats
This commit is contained in:
Maverick Studer
parent
ef13d8ace2
commit
835e3568f6
@ -2,19 +2,14 @@ package com.iqser.red.persistence.service.v1.external.api.impl.controller;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.READ_LICENSE;
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.roles.ActionRoles.UPDATE_LICENSE;
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService.*;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService;
|
||||
import com.iqser.red.service.persistence.service.v1.api.external.resource.LicenseResource;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.Feature;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.FeatureType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.License;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
@ -28,18 +23,9 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@RequiredArgsConstructor
|
||||
public class LicenseController implements LicenseResource {
|
||||
|
||||
private static final String LICENSE_OBJECT_ID = "license/redact-manager-license.json";
|
||||
private static final String DEMO_PDFTRON_LICENSE = "demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a";
|
||||
private static final String LICENSE_CUSTOMER = "LICENSE_CUSTOMER";
|
||||
private static final String LICENSE_START = "LICENSE_START";
|
||||
private static final String LICENSE_END = "LICENSE_END";
|
||||
private static final String LICENSE_PAGE_COUNT = "LICENSE_PAGE_COUNT";
|
||||
private static final String PDFTRON_FRONTEND_LICENSE = "PDFTRON_FRONTEND_LICENSE";
|
||||
private static final String PDFTRON_FEATURE = "pdftron";
|
||||
private static final String PROCESSING_PAGES_FEATURE = "processingPages";
|
||||
private static final String DEFAULT_PROCESSING_PAGES = "200000";
|
||||
private final StorageService storageService;
|
||||
private final Environment environment;
|
||||
private final LicenseUtilityService licenseUtilityService;
|
||||
private final FileFormatValidationService fileFormatValidationService;
|
||||
|
||||
|
||||
@Override
|
||||
@ -48,6 +34,7 @@ public class LicenseController implements LicenseResource {
|
||||
public void updateLicense(RedactionLicenseModel licenseModel) {
|
||||
|
||||
storageService.storeJSONObject(TenantContext.getTenantId(), LICENSE_OBJECT_ID, licenseModel);
|
||||
fileFormatValidationService.evictValidFileFormatsForTenant(TenantContext.getTenantId());
|
||||
}
|
||||
|
||||
|
||||
@ -55,80 +42,16 @@ public class LicenseController implements LicenseResource {
|
||||
@PreAuthorize("hasAuthority('" + READ_LICENSE + "')")
|
||||
public RedactionLicenseModel getLicense() {
|
||||
|
||||
if (storageService.objectExists(TenantContext.getTenantId(), LICENSE_OBJECT_ID)) {
|
||||
String tenantId = TenantContext.getTenantId();
|
||||
if (storageService.objectExists(tenantId, LICENSE_OBJECT_ID)) {
|
||||
try {
|
||||
return storageService.readJSONObject(TenantContext.getTenantId(), LICENSE_OBJECT_ID, RedactionLicenseModel.class);
|
||||
return storageService.readJSONObject(tenantId, LICENSE_OBJECT_ID, RedactionLicenseModel.class);
|
||||
} catch (Exception e) {
|
||||
return generateDemoLicense();
|
||||
return licenseUtilityService.generateDemoLicense();
|
||||
}
|
||||
} else {
|
||||
return generateDemoLicense();
|
||||
return licenseUtilityService.generateDemoLicense();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private RedactionLicenseModel generateDemoLicense() {
|
||||
|
||||
License demo = new License();
|
||||
demo.setId("RedactManager");
|
||||
demo.setName("RedactManager License");
|
||||
demo.setProduct("RedactManager");
|
||||
|
||||
var licensedTo = environment.getProperty(LICENSE_CUSTOMER, "RedactManager Demo License");
|
||||
demo.setLicensedTo(licensedTo);
|
||||
|
||||
var licenseStartDate = parseDate(environment.getProperty(LICENSE_START));
|
||||
var start = licenseStartDate != null ? licenseStartDate : OffsetDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).truncatedTo(ChronoUnit.SECONDS);
|
||||
demo.setValidFrom(start);
|
||||
|
||||
var licenseEndDate = parseDate(environment.getProperty(LICENSE_END));
|
||||
var end = licenseEndDate != null ? licenseEndDate : OffsetDateTime.now()
|
||||
.withMonth(12)
|
||||
.withDayOfMonth(31)
|
||||
.withHour(0)
|
||||
.withMinute(0)
|
||||
.withSecond(0)
|
||||
.truncatedTo(ChronoUnit.SECONDS);
|
||||
demo.setValidUntil(end);
|
||||
|
||||
var pdftronLicense = environment.getProperty(PDFTRON_FRONTEND_LICENSE, DEMO_PDFTRON_LICENSE);
|
||||
demo.getFeatures().add(Feature.builder().name(PDFTRON_FEATURE).type(FeatureType.STRING).value(pdftronLicense).build());
|
||||
|
||||
var pageCount = Long.parseLong(environment.getProperty(LICENSE_PAGE_COUNT, DEFAULT_PROCESSING_PAGES));
|
||||
demo.getFeatures().add(Feature.builder().name(PROCESSING_PAGES_FEATURE).type(FeatureType.NUMBER).value(pageCount).build());
|
||||
|
||||
var demoLicense = new RedactionLicenseModel();
|
||||
demoLicense.setActiveLicense("RedactManager");
|
||||
demoLicense.getLicenses().add(demo);
|
||||
|
||||
return demoLicense;
|
||||
}
|
||||
|
||||
|
||||
private OffsetDateTime parseDate(String date) {
|
||||
|
||||
if (StringUtils.isEmpty(date)) {
|
||||
return null;
|
||||
}
|
||||
var parts = date.split("-");
|
||||
if (parts.length != 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return OffsetDateTime.now()
|
||||
.withYear(Integer.parseInt(parts[2]))
|
||||
.withMonth(Integer.parseInt(parts[1]))
|
||||
.withDayOfMonth(Integer.parseInt(parts[0]))
|
||||
.withHour(0)
|
||||
.withMinute(0)
|
||||
.withSecond(0)
|
||||
.truncatedTo(ChronoUnit.SECONDS);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse date: {}", date);
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -7,7 +7,6 @@ import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@ -22,6 +21,8 @@ import org.springframework.web.bind.annotation.RequestPart;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.FileFormatValidationService;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.NotAllowedException;
|
||||
import com.knecon.fforesight.keycloakcommons.security.KeycloakSecurity;
|
||||
import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
|
||||
@ -34,6 +35,7 @@ import com.iqser.red.service.persistence.service.v1.api.external.resource.Upload
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AuditCategory;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileUploadResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.audit.AuditRequest;
|
||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||
|
||||
import feign.FeignException;
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
@ -52,12 +54,11 @@ public class UploadController implements UploadResource {
|
||||
private static final int THRESHOLD_SIZE = 1000000000; // 1 GB
|
||||
private static final double THRESHOLD_RATIO = 10;
|
||||
|
||||
private static final List<String> VALID_FILE_EXTENSIONS = List.of("pdf", "docx", "doc", "xls", "xlsx", "ppt", "pptx");
|
||||
|
||||
private final UploadService uploadService;
|
||||
private final ReanalysisService reanalysisService;
|
||||
private final AccessControlService accessControlService;
|
||||
private final AuditPersistenceService auditPersistenceService;
|
||||
private final FileFormatValidationService fileFormatValidationService;
|
||||
|
||||
|
||||
@Timed
|
||||
@ -79,11 +80,13 @@ public class UploadController implements UploadResource {
|
||||
case "csv":
|
||||
return uploadService.importCsv(dossierId, file.getBytes());
|
||||
default:
|
||||
if (VALID_FILE_EXTENSIONS.contains(extension)) {
|
||||
return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions);
|
||||
} else {
|
||||
if (!fileFormatValidationService.getAllFileFormats().contains(extension)) {
|
||||
throw new BadRequestException("Invalid file uploaded");
|
||||
}
|
||||
if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
|
||||
throw new NotAllowedException("Insufficient permissions");
|
||||
}
|
||||
return uploadService.processSingleFile(dossierId, file.getOriginalFilename(), file.getBytes(), keepManualRedactions);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new BadRequestException(e.getMessage(), e);
|
||||
@ -146,6 +149,12 @@ public class UploadController implements UploadResource {
|
||||
log.debug("CSV file inside ZIP failed", e);
|
||||
// TODO return un-processed files to client
|
||||
}
|
||||
} else if (zipData.fileUploadResult.getFileIds().isEmpty()) {
|
||||
if (zipData.containedUnpermittedFiles) {
|
||||
throw new NotAllowedException("Zip file contains unpermitted files");
|
||||
} else {
|
||||
throw new BadRequestException("Only unsupported files in zip file");
|
||||
}
|
||||
}
|
||||
|
||||
return zipData.fileUploadResult;
|
||||
@ -194,6 +203,7 @@ public class UploadController implements UploadResource {
|
||||
private void processFileZipEntry(ZipArchiveEntry ze, ZipFile zipFile, String dossierId, boolean keepManualRedactions, ZipData zipData) throws IOException {
|
||||
|
||||
var extension = getExtension(ze.getName());
|
||||
|
||||
final String fileName;
|
||||
if (ze.getName().lastIndexOf("/") >= 0) {
|
||||
fileName = ze.getName().substring(ze.getName().lastIndexOf("/") + 1);
|
||||
@ -216,7 +226,14 @@ public class UploadController implements UploadResource {
|
||||
|
||||
if ("csv".equals(extension)) {
|
||||
zipData.csvBytes = entryAsBytes;
|
||||
} else if (VALID_FILE_EXTENSIONS.contains(extension)) {
|
||||
} else if (fileFormatValidationService.getAllFileFormats().contains(extension)) {
|
||||
|
||||
if (!fileFormatValidationService.getValidFileFormatsForTenant(TenantContext.getTenantId()).contains(extension)) {
|
||||
zipData.containedUnpermittedFiles = true;
|
||||
return;
|
||||
}
|
||||
zipData.containedUnpermittedFiles = false;
|
||||
|
||||
try {
|
||||
var result = uploadService.processSingleFile(dossierId, fileName, entryAsBytes, keepManualRedactions);
|
||||
zipData.fileUploadResult.getFileIds().addAll(result.getFileIds());
|
||||
@ -260,6 +277,7 @@ public class UploadController implements UploadResource {
|
||||
int totalSizeArchive;
|
||||
int totalEntryArchive;
|
||||
FileUploadResult fileUploadResult = new FileUploadResult();
|
||||
boolean containedUnpermittedFiles;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,93 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.service.LicenseUtilityService.*;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.Feature;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.License;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class FileFormatValidationService {
|
||||
|
||||
final StorageService storageService;
|
||||
|
||||
final LicenseUtilityService licenseUtilityService;
|
||||
|
||||
static Map<String, List<String>> validFileFormatsCache = new HashMap<>();
|
||||
|
||||
|
||||
public List<String> getAllFileFormats() {
|
||||
|
||||
return ALL_FILE_FORMATS;
|
||||
}
|
||||
|
||||
|
||||
public List<String> getValidFileFormatsForTenant(String tenantId) {
|
||||
|
||||
boolean supportMSOfficeFormats = false;
|
||||
|
||||
if (validFileFormatsCache.containsKey(tenantId)) {
|
||||
return validFileFormatsCache.get(tenantId);
|
||||
|
||||
} else {
|
||||
Optional<License> optionalLicense = getActiveLicense(tenantId);
|
||||
if (optionalLicense.isPresent()) {
|
||||
Optional<Feature> optionalSupportMSOfficeFormatsFeature = optionalLicense.get()
|
||||
.getFeatures()
|
||||
.stream()
|
||||
.filter(f -> f.getName().equals(SUPPORT_MS_OFFICE_FORMATS_FEATURE))
|
||||
.findFirst();
|
||||
|
||||
supportMSOfficeFormats = optionalSupportMSOfficeFormatsFeature.isPresent() ? (Boolean) optionalSupportMSOfficeFormatsFeature.get().getValue() : false;
|
||||
}
|
||||
List<String> validFileFormats = supportMSOfficeFormats ? ALL_FILE_FORMATS : FILE_FORMATS_EXCLUDING_MS_OFFICE;
|
||||
|
||||
validFileFormatsCache.put(tenantId, validFileFormats);
|
||||
return validFileFormats;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void evictValidFileFormatsForTenant(String tenantId) {
|
||||
|
||||
validFileFormatsCache.remove(tenantId);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private Optional<License> getActiveLicense(String tenantId) {
|
||||
|
||||
var redactionLicenseModel = getLicenseForTenant(tenantId);
|
||||
return redactionLicenseModel.getLicenses().stream().filter(rlm -> Objects.equals(rlm.getId(), redactionLicenseModel.getActiveLicense())).findFirst();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private RedactionLicenseModel getLicenseForTenant(String tenantId) {
|
||||
|
||||
if (storageService.objectExists(tenantId, LICENSE_OBJECT_ID)) {
|
||||
try {
|
||||
return storageService.readJSONObject(tenantId, LICENSE_OBJECT_ID, RedactionLicenseModel.class);
|
||||
} catch (Exception e) {
|
||||
return licenseUtilityService.generateDemoLicense();
|
||||
}
|
||||
}
|
||||
return licenseUtilityService.generateDemoLicense();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,111 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.Feature;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.FeatureType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.License;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.license.RedactionLicenseModel;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@AllArgsConstructor
|
||||
public class LicenseUtilityService {
|
||||
|
||||
public static final String LICENSE_OBJECT_ID = "license/redact-manager-license.json";
|
||||
public static final String DEMO_PDFTRON_LICENSE = "demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a";
|
||||
public static final String LICENSE_CUSTOMER = "LICENSE_CUSTOMER";
|
||||
public static final String LICENSE_START = "LICENSE_START";
|
||||
public static final String LICENSE_END = "LICENSE_END";
|
||||
public static final String LICENSE_PAGE_COUNT = "LICENSE_PAGE_COUNT";
|
||||
public static final String LICENSE_MS_OFFICE_FORMATS = "LICENSE_SUPPORT_MS_OFFICE_FORMATS";
|
||||
public static final String PDFTRON_FRONTEND_LICENSE = "PDFTRON_FRONTEND_LICENSE";
|
||||
public static final String PDFTRON_FEATURE = "pdftron";
|
||||
public static final String PROCESSING_PAGES_FEATURE = "processingPages";
|
||||
public static final String DEFAULT_PROCESSING_PAGES = "200000";
|
||||
public static final String SUPPORT_MS_OFFICE_FORMATS_FEATURE = "supportMSOfficeFormats";
|
||||
public static final String DEFAULT_SUPPORT_MS_OFFICE_FORMATS = "false";
|
||||
|
||||
public static final List<String> ALL_FILE_FORMATS = List.of("pdf", "docx", "doc", "xls", "xlsx", "ppt", "pptx");
|
||||
public static final List<String> FILE_FORMATS_EXCLUDING_MS_OFFICE = List.of("pdf");
|
||||
|
||||
private final Environment environment;
|
||||
|
||||
|
||||
public RedactionLicenseModel generateDemoLicense() {
|
||||
|
||||
License demo = new License();
|
||||
demo.setId("RedactManager");
|
||||
demo.setName("RedactManager License");
|
||||
demo.setProduct("RedactManager");
|
||||
|
||||
var licensedTo = environment.getProperty(LICENSE_CUSTOMER, "RedactManager Demo License");
|
||||
demo.setLicensedTo(licensedTo);
|
||||
|
||||
var licenseStartDate = parseDate(environment.getProperty(LICENSE_START));
|
||||
var start = licenseStartDate != null ? licenseStartDate : OffsetDateTime.now().withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).truncatedTo(ChronoUnit.SECONDS);
|
||||
demo.setValidFrom(start);
|
||||
|
||||
var licenseEndDate = parseDate(environment.getProperty(LICENSE_END));
|
||||
var end = licenseEndDate != null ? licenseEndDate : OffsetDateTime.now()
|
||||
.withMonth(12)
|
||||
.withDayOfMonth(31)
|
||||
.withHour(0)
|
||||
.withMinute(0)
|
||||
.withSecond(0)
|
||||
.truncatedTo(ChronoUnit.SECONDS);
|
||||
demo.setValidUntil(end);
|
||||
|
||||
var pdftronLicense = environment.getProperty(PDFTRON_FRONTEND_LICENSE, DEMO_PDFTRON_LICENSE);
|
||||
demo.getFeatures().add(Feature.builder().name(PDFTRON_FEATURE).type(FeatureType.STRING).value(pdftronLicense).build());
|
||||
|
||||
var pageCount = Long.parseLong(environment.getProperty(LICENSE_PAGE_COUNT, DEFAULT_PROCESSING_PAGES));
|
||||
demo.getFeatures().add(Feature.builder().name(PROCESSING_PAGES_FEATURE).type(FeatureType.NUMBER).value(pageCount).build());
|
||||
|
||||
boolean supportMSOfficeFormats = Boolean.parseBoolean(environment.getProperty(LICENSE_MS_OFFICE_FORMATS, DEFAULT_SUPPORT_MS_OFFICE_FORMATS));
|
||||
demo.getFeatures().add(Feature.builder().name(SUPPORT_MS_OFFICE_FORMATS_FEATURE).type(FeatureType.BOOLEAN).value(supportMSOfficeFormats).build());
|
||||
|
||||
var demoLicense = new RedactionLicenseModel();
|
||||
demoLicense.setActiveLicense("RedactManager");
|
||||
demoLicense.getLicenses().add(demo);
|
||||
|
||||
return demoLicense;
|
||||
}
|
||||
|
||||
|
||||
private OffsetDateTime parseDate(String date) {
|
||||
|
||||
if (StringUtils.isEmpty(date)) {
|
||||
return null;
|
||||
}
|
||||
var parts = date.split("-");
|
||||
if (parts.length != 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return OffsetDateTime.now()
|
||||
.withYear(Integer.parseInt(parts[2]))
|
||||
.withMonth(Integer.parseInt(parts[1]))
|
||||
.withDayOfMonth(Integer.parseInt(parts[0]))
|
||||
.withHour(0)
|
||||
.withMinute(0)
|
||||
.withSecond(0)
|
||||
.truncatedTo(ChronoUnit.SECONDS);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse date: {}", date);
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user