RED-4645: Multitenancy for storage

This commit is contained in:
deiflaender 2023-03-10 16:30:19 +01:00
parent 5003970fe1
commit e5df0ec658
17 changed files with 388 additions and 27 deletions

View File

@ -14,7 +14,7 @@
<version>1.0-SNAPSHOT</version>
<properties>
<persistence-service.version>2.0.8</persistence-service.version>
<persistence-service.version>2.1.0</persistence-service.version>
<dsljson.version>1.9.9</dsljson.version>
</properties>

View File

@ -10,9 +10,11 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.scheduling.annotation.EnableAsync;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.ocr.v1.server.multitenancy.AsyncConfig;
import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
import com.iqser.red.service.ocr.v1.server.multitenancy.MultiTenancyWebConfiguration;
import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
import io.micrometer.core.aop.TimedAspect;
@ -21,7 +23,7 @@ import io.micrometer.core.instrument.MeterRegistry;
@EnableAsync
@EnableConfigurationProperties(OcrServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@Import({DefaultWebMvcConfiguration.class, MessagingConfiguration.class})
@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MessagingConfiguration.class, MultiTenancyMessagingConfiguration.class})
@EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
public class Application {

View File

@ -0,0 +1,10 @@
package com.iqser.red.service.ocr.v1.server.client;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
public interface TenantsClient extends TenantsResource {
}

View File

@ -0,0 +1,27 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import java.util.concurrent.Executor;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
public class AsyncConfig extends AsyncConfigurerSupport {
@Override
public Executor getAsyncExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(7);
executor.setMaxPoolSize(42);
executor.setQueueCapacity(11);
executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
executor.setTaskDecorator(new TenantAwareTaskDecorator());
executor.initialize();
return executor;
}
}

View File

@ -0,0 +1,105 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import java.util.Base64;
import javax.annotation.PostConstruct;
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import lombok.SneakyThrows;
@Service
public class EncryptionDecryptionService {
@Value("${ocr-service.crypto.key:redaction}")
private String key;
private SecretKey secretKey;
private byte[] iv;
@SneakyThrows
@PostConstruct
protected void postConstruct() {
SecureRandom secureRandom = new SecureRandom();
iv = new byte[12];
secureRandom.nextBytes(iv);
secretKey = generateSecretKey(key, iv);
}
@SneakyThrows
public String encrypt(String strToEncrypt) {
return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
}
@SneakyThrows
public String decrypt(String strToDecrypt) {
byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
return new String(decrypt(bytes), StandardCharsets.UTF_8);
}
@SneakyThrows
public byte[] encrypt(byte[] data) {
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
byte[] encryptedData = cipher.doFinal(data);
ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
byteBuffer.putInt(iv.length);
byteBuffer.put(iv);
byteBuffer.put(encryptedData);
return byteBuffer.array();
}
@SneakyThrows
public byte[] decrypt(byte[] encryptedData) {
ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
int noonceSize = byteBuffer.getInt();
if (noonceSize < 12 || noonceSize >= 16) {
throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
}
byte[] iv = new byte[noonceSize];
byteBuffer.get(iv);
SecretKey secretKey = generateSecretKey(key, iv);
byte[] cipherBytes = new byte[byteBuffer.remaining()];
byteBuffer.get(cipherBytes);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
return cipher.doFinal(cipherBytes);
}
@SneakyThrows
public SecretKey generateSecretKey(String password, byte[] iv) {
KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
return new SecretKeySpec(key, "AES");
}
}

View File

@ -0,0 +1,17 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.stereotype.Component;
import feign.RequestInterceptor;
import feign.RequestTemplate;
@Component
public class ForwardTenantInterceptor implements RequestInterceptor {
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
@Override
public void apply(RequestTemplate template) {
template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
}
}

View File

@ -0,0 +1,48 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import static com.iqser.red.service.ocr.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class MultiTenancyMessagingConfiguration {
@Bean
public static BeanPostProcessor multitenancyBeanPostProcessor() {
return new BeanPostProcessor() {
@Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
if (bean instanceof RabbitTemplate) {
((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
return m;
});
} else if (bean instanceof AbstractRabbitListenerContainerFactory) {
((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
if (tenant != null) {
TenantContext.setTenantId(tenant);
} else {
throw new RuntimeException("No Tenant is set queue message");
}
return m;
});
}
return bean;
}
};
}
}

View File

@ -0,0 +1,28 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
@Configuration
public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
private final TenantInterceptor tenantInterceptor;
@Autowired
public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
this.tenantInterceptor = tenantInterceptor;
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addWebRequestInterceptor(tenantInterceptor);
}
}

View File

@ -0,0 +1,45 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.stereotype.Service;
import com.iqser.red.service.ocr.v1.server.client.TenantsClient;
import com.iqser.red.storage.commons.model.AzureStorageConnection;
import com.iqser.red.storage.commons.model.S3StorageConnection;
import com.iqser.red.storage.commons.service.StorageConnectionProvider;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class StorageConnectionProviderImpl implements StorageConnectionProvider {
private final TenantsClient tenantsClient;
private final EncryptionDecryptionService encryptionDecryptionService;
@Override
public AzureStorageConnection getAzureStorageConnection(String tenantId) {
var tenant = tenantsClient.getTenant(tenantId);
return AzureStorageConnection.builder()
.connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
.containerName(tenant.getAzureStorageConnection().getContainerName())
.build();
}
@Override
public S3StorageConnection getS3StorageConnection(String tenantId) {
var tenant = tenantsClient.getTenant(tenantId);
return S3StorageConnection.builder()
.key(tenant.getS3StorageConnection().getKey())
.secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
.signerType(tenant.getS3StorageConnection().getSignerType())
.bucketName(tenant.getS3StorageConnection().getBucketName())
.region(tenant.getS3StorageConnection().getRegion())
.endpoint(tenant.getS3StorageConnection().getEndpoint())
.build();
}
}

View File

@ -0,0 +1,23 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.core.task.TaskDecorator;
import org.springframework.lang.NonNull;
public class TenantAwareTaskDecorator implements TaskDecorator {
@Override
@NonNull
public Runnable decorate(@NonNull Runnable runnable) {
String tenantId = TenantContext.getTenantId();
return () -> {
try {
TenantContext.setTenantId(tenantId);
runnable.run();
} finally {
TenantContext.setTenantId(null);
}
};
}
}

View File

@ -0,0 +1,29 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public final class TenantContext {
private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
public static void setTenantId(String tenantId) {
log.debug("Setting tenantId to " + tenantId);
currentTenant.set(tenantId);
}
public static String getTenantId() {
return currentTenant.get();
}
public static void clear() {
currentTenant.remove();
}
}

View File

@ -0,0 +1,35 @@
package com.iqser.red.service.ocr.v1.server.multitenancy;
import org.springframework.stereotype.Component;
import org.springframework.ui.ModelMap;
import org.springframework.web.context.request.WebRequest;
import org.springframework.web.context.request.WebRequestInterceptor;
@Component
public class TenantInterceptor implements WebRequestInterceptor {
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
@Override
public void preHandle(WebRequest request) {
if (request.getHeader(TENANT_HEADER_NAME) != null) {
TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
}
}
@Override
public void postHandle(WebRequest request, ModelMap model) {
TenantContext.clear();
}
@Override
public void afterCompletion(WebRequest request, Exception ex) {
}
}

View File

@ -7,6 +7,7 @@ import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.ocr.v1.server.model.image.ImageServiceResponse;
import com.iqser.red.service.ocr.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.storage.commons.service.StorageService;
@ -31,39 +32,39 @@ public class FileStorageService {
@SneakyThrows
public byte[] getOriginalFile(String dossierId, String fileId) {
return IOUtils.toByteArray(storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
}
@SneakyThrows
public InputStream getOriginalFileAsStream(String dossierId, String fileId) {
return storageService.getObject(getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
return storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
}
public void storeOriginalFile(String dossierId, String fileId, InputStream stream) {
storageService.storeObject(getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN), stream);
}
public boolean untouchedFileExists(String dossierId, String fileId) {
return storageService.objectExists(getStorageId(dossierId, fileId, FileType.UNTOUCHED));
return storageService.objectExists(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED));
}
public void storeUntouchedFile(String dossierId, String fileId, byte[] data) {
storageService.storeObject(getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
storageService.storeObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.UNTOUCHED), new ByteArrayInputStream(data));
}
@SneakyThrows
public ImageServiceResponse getImageServiceResponse(String dossierId, String fileId) {
return storageService.readJSONObject(getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
return storageService.readJSONObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
}
}

View File

@ -40,10 +40,6 @@ management:
storage:
signer-type: 'AWSS3V4SignerType'
bucket-name: 'redaction'
region: 'us-east-1'
endpoint: 'https://s3.amazonaws.com'
backend: 's3'
pdftron.license: ${PDFTRON_LICENSE}

View File

@ -16,6 +16,7 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.ocr.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.ocr.v1.server.service.FileStorageService;
import com.iqser.red.service.ocr.v1.server.service.OCRService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@ -118,7 +119,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
var originId = FileStorageService.getStorageId("dossier", "file", FileType.ORIGIN);
try (var fileStream = pdfFileResource.getInputStream()) {
storageService.storeObject(originId, fileStream);
storageService.storeObject(TenantContext.getTenantId(), originId, fileStream);
}
try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
ocrService.runOcrOnDocument("dossier", "file", out);

View File

@ -32,7 +32,7 @@ public class FileSystemBackedStorageService implements StorageService {
@SneakyThrows
@Override
public InputStreamResource getObject(String objectId) {
public InputStreamResource getObject(String tenantId, String objectId) {
var res = dataMap.get(objectId);
if (res == null) {
@ -44,28 +44,22 @@ public class FileSystemBackedStorageService implements StorageService {
@Override
public void deleteObject(String objectId) {
public void deleteObject(String tenantId, String objectId) {
dataMap.remove(objectId);
}
@Override
public boolean objectExists(String objectId) {
public boolean objectExists(String tenantId, String objectId) {
return dataMap.containsKey(objectId);
}
@Override
public void init() {
}
@Override
@SneakyThrows
public <T> void storeJSONObject(String objectId, T any) {
public <T> void storeJSONObject(String tenantId, String objectId, T any) {
File tempFile = File.createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any);
@ -81,7 +75,7 @@ public class FileSystemBackedStorageService implements StorageService {
@Override
@SneakyThrows
public <T> T readJSONObject(String objectId, Class<T> clazz) {
public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
throw new StorageObjectDoesNotExist("Stored object not found");
@ -104,7 +98,7 @@ public class FileSystemBackedStorageService implements StorageService {
@Override
@SneakyThrows
public void storeObject(String objectId, InputStream stream) {
public void storeObject(String tenantId, String objectId, InputStream stream) {
File tempFile = File.createTempFile("test", ".tmp");

View File

@ -27,7 +27,7 @@
<dependency>
<groupId>com.iqser.red</groupId>
<artifactId>platform-commons-dependency</artifactId>
<version>1.21.0</version>
<version>1.22.0</version>
<scope>import</scope>
<type>pom</type>
</dependency>