From c13d148ae7bc4a8f9e4d9355fc93e66e97603eac Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 10 Mar 2023 16:21:54 +0100 Subject: [PATCH] RED-4645: Multitenancy for storage --- redaction-service-v1/pom.xml | 2 +- .../redaction-service-api-v1/pom.xml | 2 +- .../redaction/v1/server/Application.java | 5 +- .../v1/server/client/TenantsClient.java | 10 ++ .../v1/server/multitenancy/AsyncConfig.java | 27 +++++ .../EncryptionDecryptionService.java | 105 ++++++++++++++++++ .../ForwardTenantInterceptor.java | 17 +++ .../MultiTenancyMessagingConfiguration.java | 49 ++++++++ .../MultiTenancyWebConfiguration.java | 28 +++++ .../StorageConnectionProviderImpl.java | 45 ++++++++ .../TenantAwareTaskDecorator.java | 23 ++++ .../v1/server/multitenancy/TenantContext.java | 29 +++++ .../multitenancy/TenantInterceptor.java | 35 ++++++ .../storage/RedactionStorageService.java | 17 +-- .../src/main/resources/application.yml | 4 - .../FileSystemBackedStorageService.java | 18 +-- .../HeadlinesGoldStandardIntegrationTest.java | 7 +- .../v1/server/RedactionIntegrationTest.java | 11 +- .../redaction/v1/server/RulesTest.java | 7 +- .../PdfSegmentationServiceTest.java | 3 +- .../src/test/resources/application.yml | 4 - 21 files changed, 405 insertions(+), 43 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/TenantsClient.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/AsyncConfig.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/ForwardTenantInterceptor.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyWebConfiguration.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/StorageConnectionProviderImpl.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantAwareTaskDecorator.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantContext.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantInterceptor.java diff --git a/redaction-service-v1/pom.xml b/redaction-service-v1/pom.xml index 6feb459a..f28c2d73 100644 --- a/redaction-service-v1/pom.xml +++ b/redaction-service-v1/pom.xml @@ -32,7 +32,7 @@ com.iqser.red platform-commons-dependency - 1.20.0 + 1.22.0 import pom diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index 1dece7ae..ee0e781a 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -12,7 +12,7 @@ redaction-service-api-v1 - 2.0.8 + 2.1.0 diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java index 51880f15..cfd44b4c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java @@ -11,12 +11,15 @@ import org.springframework.context.annotation.Import; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig; +import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration; +import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyWebConfiguration; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.instrument.MeterRegistry; -@Import({DefaultWebMvcConfiguration.class}) +@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class}) @EnableFeignClients(basePackageClasses = RulesClient.class) @EnableConfigurationProperties(RedactionServiceSettings.class) @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/TenantsClient.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/TenantsClient.java new file mode 100644 index 00000000..c5a016c0 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/TenantsClient.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.redaction.v1.server.client; + +import org.springframework.cloud.openfeign.FeignClient; + +import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource; + +@FeignClient(name = "TenantsResource", url = "${persistence-service.url}") +public interface TenantsClient extends TenantsResource { + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/AsyncConfig.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/AsyncConfig.java new file mode 100644 index 00000000..0b6824df --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/AsyncConfig.java @@ -0,0 +1,27 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import java.util.concurrent.Executor; + +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.AsyncConfigurerSupport; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +@Configuration +public class AsyncConfig extends AsyncConfigurerSupport { + + @Override + public Executor getAsyncExecutor() { + + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + + executor.setCorePoolSize(7); + executor.setMaxPoolSize(42); + executor.setQueueCapacity(11); + executor.setThreadNamePrefix("TenantAwareTaskExecutor-"); + executor.setTaskDecorator(new TenantAwareTaskDecorator()); + executor.initialize(); + + return executor; + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java new file mode 100644 index 00000000..26d5acdd --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/EncryptionDecryptionService.java @@ -0,0 +1,105 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.security.spec.KeySpec; +import java.util.Base64; + +import javax.annotation.PostConstruct; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import lombok.SneakyThrows; + +@Service +public class EncryptionDecryptionService { + + @Value("${redaction-service.crypto.key:redaction}") + private String key; + + private SecretKey secretKey; + private byte[] iv; + + + @SneakyThrows + @PostConstruct + protected void postConstruct() { + + SecureRandom secureRandom = new SecureRandom(); + iv = new byte[12]; + secureRandom.nextBytes(iv); + secretKey = generateSecretKey(key, iv); + } + + + @SneakyThrows + public String encrypt(String strToEncrypt) { + + return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes())); + } + + + @SneakyThrows + public String decrypt(String strToDecrypt) { + + byte[] bytes = Base64.getDecoder().decode(strToDecrypt); + return new String(decrypt(bytes), StandardCharsets.UTF_8); + } + + + @SneakyThrows + public byte[] encrypt(byte[] data) { + + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec); + byte[] encryptedData = cipher.doFinal(data); + ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length); + byteBuffer.putInt(iv.length); + byteBuffer.put(iv); + byteBuffer.put(encryptedData); + return byteBuffer.array(); + } + + + @SneakyThrows + public byte[] decrypt(byte[] encryptedData) { + + ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData); + int noonceSize = byteBuffer.getInt(); + if (noonceSize < 12 || noonceSize >= 16) { + throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file."); + } + byte[] iv = new byte[noonceSize]; + byteBuffer.get(iv); + + SecretKey secretKey = generateSecretKey(key, iv); + + byte[] cipherBytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(cipherBytes); + + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); + cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec); + return cipher.doFinal(cipherBytes); + } + + + @SneakyThrows + public SecretKey generateSecretKey(String password, byte[] iv) { + + KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128 + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); + byte[] key = secretKeyFactory.generateSecret(spec).getEncoded(); + return new SecretKeySpec(key, "AES"); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/ForwardTenantInterceptor.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/ForwardTenantInterceptor.java new file mode 100644 index 00000000..40f83bc4 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/ForwardTenantInterceptor.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import org.springframework.stereotype.Component; + +import feign.RequestInterceptor; +import feign.RequestTemplate; + +@Component +public class ForwardTenantInterceptor implements RequestInterceptor { + + public static final String TENANT_HEADER_NAME = "X-TENANT-ID"; + + @Override + public void apply(RequestTemplate template) { + template.header(TENANT_HEADER_NAME, TenantContext.getTenantId()); + } +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java new file mode 100644 index 00000000..dd2913e5 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java @@ -0,0 +1,49 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + + +import static com.iqser.red.service.redaction.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME; + +import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MultiTenancyMessagingConfiguration { + + @Bean + public static BeanPostProcessor multitenancyBeanPostProcessor() { + + return new BeanPostProcessor() { + + @Override + public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { + + if (bean instanceof RabbitTemplate) { + + ((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> { + m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId()); + return m; + }); + + } else if (bean instanceof AbstractRabbitListenerContainerFactory) { + + ((AbstractRabbitListenerContainerFactory) bean).setAfterReceivePostProcessors(m -> { + String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME); + + if (tenant != null) { + TenantContext.setTenantId(tenant); + } else { + throw new RuntimeException("No Tenant is set queue message"); + } + return m; + }); + } + return bean; + } + }; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyWebConfiguration.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyWebConfiguration.java new file mode 100644 index 00000000..a1683fa8 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/MultiTenancyWebConfiguration.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; + +import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; + +@Configuration +public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration { + + private final TenantInterceptor tenantInterceptor; + + + @Autowired + public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) { + + this.tenantInterceptor = tenantInterceptor; + } + + + @Override + public void addInterceptors(InterceptorRegistry registry) { + + registry.addWebRequestInterceptor(tenantInterceptor); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/StorageConnectionProviderImpl.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/StorageConnectionProviderImpl.java new file mode 100644 index 00000000..5199b13c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/StorageConnectionProviderImpl.java @@ -0,0 +1,45 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.redaction.v1.server.client.TenantsClient; +import com.iqser.red.storage.commons.model.AzureStorageConnection; +import com.iqser.red.storage.commons.model.S3StorageConnection; +import com.iqser.red.storage.commons.service.StorageConnectionProvider; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class StorageConnectionProviderImpl implements StorageConnectionProvider { + + private final TenantsClient tenantsClient; + private final EncryptionDecryptionService encryptionDecryptionService; + + + @Override + public AzureStorageConnection getAzureStorageConnection(String tenantId) { + + var tenant = tenantsClient.getTenant(tenantId); + return AzureStorageConnection.builder() + .connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString())) + .containerName(tenant.getAzureStorageConnection().getContainerName()) + .build(); + } + + + @Override + public S3StorageConnection getS3StorageConnection(String tenantId) { + + var tenant = tenantsClient.getTenant(tenantId); + return S3StorageConnection.builder() + .key(tenant.getS3StorageConnection().getKey()) + .secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret())) + .signerType(tenant.getS3StorageConnection().getSignerType()) + .bucketName(tenant.getS3StorageConnection().getBucketName()) + .region(tenant.getS3StorageConnection().getRegion()) + .endpoint(tenant.getS3StorageConnection().getEndpoint()) + .build(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantAwareTaskDecorator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantAwareTaskDecorator.java new file mode 100644 index 00000000..804b5de3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantAwareTaskDecorator.java @@ -0,0 +1,23 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import org.springframework.core.task.TaskDecorator; +import org.springframework.lang.NonNull; + +public class TenantAwareTaskDecorator implements TaskDecorator { + + @Override + @NonNull + public Runnable decorate(@NonNull Runnable runnable) { + + String tenantId = TenantContext.getTenantId(); + return () -> { + try { + TenantContext.setTenantId(tenantId); + runnable.run(); + } finally { + TenantContext.setTenantId(null); + } + }; + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantContext.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantContext.java new file mode 100644 index 00000000..5ef701ff --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantContext.java @@ -0,0 +1,29 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public final class TenantContext { + + private static InheritableThreadLocal currentTenant = new InheritableThreadLocal<>(); + + + public static void setTenantId(String tenantId) { + + log.debug("Setting tenantId to " + tenantId); + currentTenant.set(tenantId); + } + + + public static String getTenantId() { + + return currentTenant.get(); + } + + + public static void clear() { + + currentTenant.remove(); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantInterceptor.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantInterceptor.java new file mode 100644 index 00000000..6d5e7b67 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/multitenancy/TenantInterceptor.java @@ -0,0 +1,35 @@ +package com.iqser.red.service.redaction.v1.server.multitenancy; + +import org.springframework.stereotype.Component; +import org.springframework.ui.ModelMap; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.context.request.WebRequestInterceptor; + +@Component +public class TenantInterceptor implements WebRequestInterceptor { + + public static final String TENANT_HEADER_NAME = "X-TENANT-ID"; + + + @Override + public void preHandle(WebRequest request) { + + if (request.getHeader(TENANT_HEADER_NAME) != null) { + TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME)); + } + } + + + @Override + public void postHandle(WebRequest request, ModelMap model) { + + TenantContext.clear(); + } + + + @Override + public void afterCompletion(WebRequest request, Exception ex) { + + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 3bc1770d..2bcbd9cd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -11,6 +11,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; @@ -31,14 +32,14 @@ public class RedactionStorageService { @SneakyThrows public InputStream getStoredObject(String storageId) { - return storageService.getObject(storageId).getInputStream(); + return storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream(); } @SneakyThrows public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) { - storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream); + storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream); } @@ -46,7 +47,7 @@ public class RedactionStorageService { @Timed("redactmanager_storeObject") public void storeObject(String dossierId, String fileId, FileType fileType, Object any) { - storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any); + storageService.storeJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, fileType), any); } @@ -54,7 +55,7 @@ public class RedactionStorageService { public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { try { - return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class); + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class); } catch (StorageObjectDoesNotExist e) { log.debug("Imported redactions not available."); return null; @@ -66,7 +67,7 @@ public class RedactionStorageService { public RedactionLog getRedactionLog(String dossierId, String fileId) { try { - return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class); + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class); } catch (StorageObjectDoesNotExist e) { log.debug("Text not available."); return null; @@ -79,7 +80,7 @@ public class RedactionStorageService { public Text getText(String dossierId, String fileId) { try { - return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class); + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), Text.class); } catch (StorageObjectDoesNotExist e) { log.debug("Text not available."); return null; @@ -91,7 +92,7 @@ public class RedactionStorageService { public NerEntities getNerEntities(String dossierId, String fileId) { try { - return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class); + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class); } catch (StorageObjectDoesNotExist e) { throw new NotFoundException("NER Entities are not available."); } @@ -102,7 +103,7 @@ public class RedactionStorageService { public SectionGrid getSectionGrid(String dossierId, String fileId) { try { - return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class); + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class); } catch (StorageObjectDoesNotExist e) { throw new NotFoundException("Section Grid is not available."); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml index 69096dd2..4d8de8e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml @@ -38,8 +38,4 @@ management: storage: - signer-type: 'AWSS3V4SignerType' - bucket-name: 'redaction' - region: 'us-east-1' - endpoint: 'https://s3.amazonaws.com' backend: 's3' diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java index 16dc95d1..7e640ce8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java @@ -33,7 +33,7 @@ public class FileSystemBackedStorageService implements StorageService { @SneakyThrows @Override - public InputStreamResource getObject(String objectId) { + public InputStreamResource getObject(String tenantId, String objectId) { var res = dataMap.get(objectId); if (res == null) { @@ -45,28 +45,22 @@ public class FileSystemBackedStorageService implements StorageService { @Override - public void deleteObject(String objectId) { + public void deleteObject(String tenantId, String objectId) { dataMap.remove(objectId); } @Override - public boolean objectExists(String objectId) { + public boolean objectExists(String tenantId, String objectId) { return dataMap.containsKey(objectId); } - @Override - public void init() { - - } - - @Override @SneakyThrows - public void storeJSONObject(String objectId, T any) { + public void storeJSONObject(String tenantId, String objectId, T any) { File tempFile = FileUtils.createTempFile("test", ".tmp"); getMapper().writeValue(new FileOutputStream(tempFile), any); @@ -82,7 +76,7 @@ public class FileSystemBackedStorageService implements StorageService { @Override @SneakyThrows - public T readJSONObject(String objectId, Class clazz) { + public T readJSONObject(String tenantId, String objectId, Class clazz) { if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) { throw new StorageObjectDoesNotExist("Stored object not found"); @@ -105,7 +99,7 @@ public class FileSystemBackedStorageService implements StorageService { @Override @SneakyThrows - public void storeObject(String objectId, InputStream stream) { + public void storeObject(String tenantId, String objectId, InputStream stream) { File tempFile = FileUtils.createTempFile("test", ".tmp"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java index 83137eb2..69052f68 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java @@ -57,6 +57,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; @@ -360,8 +361,8 @@ public class HeadlinesGoldStandardIntegrationTest { .lastProcessed(OffsetDateTime.now()) .build(); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); return request; } @@ -390,7 +391,7 @@ public class HeadlinesGoldStandardIntegrationTest { private void loadNerForTest() { ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 1feb31bc..9bdbb63d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -79,6 +79,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; @@ -641,7 +642,7 @@ public class RedactionIntegrationTest { String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json"); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); long start = System.currentTimeMillis(); AnalyzeRequest request = prepareStorage(fileName); @@ -1530,7 +1531,7 @@ public class RedactionIntegrationTest { private void loadNerForTest() { ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); } @@ -1596,7 +1597,7 @@ public class RedactionIntegrationTest { ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json"); AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf"); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream()); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); @@ -1667,8 +1668,8 @@ public class RedactionIntegrationTest { .lastProcessed(OffsetDateTime.now()) .build(); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); return request; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 0f172f91..b453453c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -72,6 +72,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; @@ -618,7 +619,7 @@ public class RulesTest { private void loadNerForTest() { ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); } @@ -641,9 +642,9 @@ public class RulesTest { .lastProcessed(OffsetDateTime.now()) .build(); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), new ClassPathResource("files/cv_service_empty_response.json").getInputStream()); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream); return request; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java index 1971909d..05df483c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java @@ -35,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; @@ -199,7 +200,7 @@ public class PdfSegmentationServiceTest { @SneakyThrows private void prepareStorage() { - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), new ClassPathResource("files/cv_service_empty_response.json").getInputStream()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml index 2ff3f881..74de1579 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/application.yml @@ -21,10 +21,6 @@ redaction-service: enable-entity-recognition: true storage: - signer-type: 'AWSS3V4SignerType' - bucket-name: 'redaction' - region: 'us-east-1' - endpoint: 'https://s3.amazonaws.com' backend: 's3' management: