diff --git a/search-service-v1/pom.xml b/search-service-v1/pom.xml index 88552c8..96052ab 100644 --- a/search-service-v1/pom.xml +++ b/search-service-v1/pom.xml @@ -28,7 +28,7 @@ com.iqser.red platform-commons-dependency - 1.21.0 + 1.22.0 import pom diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/StorageConnectionProviderImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/StorageConnectionProviderImpl.java new file mode 100644 index 0000000..4cdde12 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/StorageConnectionProviderImpl.java @@ -0,0 +1,45 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.client.TenantsClient; +import com.iqser.red.storage.commons.model.AzureStorageConnection; +import com.iqser.red.storage.commons.model.S3StorageConnection; +import com.iqser.red.storage.commons.service.StorageConnectionProvider; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class StorageConnectionProviderImpl implements StorageConnectionProvider { + + private final TenantsClient tenantsClient; + private final EncryptionDecryptionService encryptionDecryptionService; + + + @Override + public AzureStorageConnection getAzureStorageConnection(String tenantId) { + + var tenant = tenantsClient.getTenant(tenantId); + return AzureStorageConnection.builder() + .connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString())) + .containerName(tenant.getAzureStorageConnection().getContainerName()) + .build(); + } + + + @Override + public S3StorageConnection getS3StorageConnection(String tenantId) { + + var tenant = tenantsClient.getTenant(tenantId); + return S3StorageConnection.builder() + .key(tenant.getS3StorageConnection().getKey()) + .secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret())) + .signerType(tenant.getS3StorageConnection().getSignerType()) + .bucketName(tenant.getS3StorageConnection().getBucketName()) + .region(tenant.getS3StorageConnection().getRegion()) + .endpoint(tenant.getS3StorageConnection().getEndpoint()) + .build(); + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java index 33aa5a0..c5e713f 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java @@ -4,6 +4,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; @@ -23,7 +24,7 @@ public class TextStorageService { public Text getText(String dossierId, String fileId) { try { - return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT), Text.class); + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SIMPLIFIED_TEXT), Text.class); } catch (StorageObjectDoesNotExist e) { throw new RuntimeException("Text is not available", e); } catch (Exception e) { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java index 616a0a0..1bd022d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java @@ -47,9 +47,20 @@ public class EsClientCache { .maximumSize(maximumSize) .expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES) .removalListener((RemovalListener) removal -> { - var elasticsearchClient = removal.getValue(); - elasticsearchClient.shutdown(); - log.info("Closed elasticsearch client for tenant {}", removal.getKey()); + var clientToRemove = removal.getValue(); + int numberOfUsersForSameClient = 0; + for (var client : clients.asMap().values()) { + if (clientToRemove.getElasticsearchClient().equals(client.getElasticsearchClient())){ + numberOfUsersForSameClient++; + } + } + if(numberOfUsersForSameClient <= 1){ + clientToRemove.shutdown(); + log.info("Closed elasticsearch client for tenant {}", removal.getKey()); + } else { + log.info("Keeping client open from {} because it is still used by {} other tenants", removal.getKey(), numberOfUsersForSameClient - 1); + } + }) .build(new CacheLoader<>() { public EsClient load(String key) { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java index 3c1fd50..878f39f 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java @@ -47,9 +47,20 @@ public class OpensearchClientCache { .maximumSize(maximumSize) .expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES) .removalListener((RemovalListener) removal -> { - var client = removal.getValue(); - client.shutdown(); - log.info("Closed opensearch client for tenant {}", removal.getKey()); + var clientToRemove = removal.getValue(); + int numberOfUsersForSameClient = 0; + for (var client : clients.asMap().values()) { + if (clientToRemove.getClient().equals(client.getClient())){ + numberOfUsersForSameClient++; + } + } + if(numberOfUsersForSameClient <= 1){ + clientToRemove.shutdown(); + log.info("Closed elasticsearch client for tenant {}", removal.getKey()); + } else { + log.info("Keeping client open from {} because it is still used by {} other tenants", removal.getKey(), numberOfUsersForSameClient - 1); + } + }) .build(new CacheLoader<>() { public OpensearchClient load(String key) { diff --git a/search-service-v1/search-service-server-v1/src/main/resources/application.yml b/search-service-v1/search-service-server-v1/src/main/resources/application.yml index fbc40e5..2a998de 100644 --- a/search-service-v1/search-service-server-v1/src/main/resources/application.yml +++ b/search-service-v1/search-service-server-v1/src/main/resources/application.yml @@ -37,18 +37,5 @@ management: search: backend: elasticsearch -elasticsearch: - hosts: - - ${elasticsearch.cluster.hosts} - port: ${elasticsearch.cluster.port:9200} - scheme: ${elasticsearch.cluster.scheme:http} - username: ${elasticsearch.cluster.username} - password: ${elasticsearch.cluster.password} - apiKeyAuth: ${elasticsearch.cluster.apikey} - storage: - signer-type: 'AWSS3V4SignerType' - bucket-name: 'redaction' - region: 'us-east-1' - endpoint: 'https://s3.amazonaws.com' backend: 's3' diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java index 5c3c02e..ffc5799 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java @@ -32,7 +32,7 @@ public class FileSystemBackedStorageService implements StorageService { @SneakyThrows @Override - public InputStreamResource getObject(String objectId) { + public InputStreamResource getObject(String tenantId, String objectId) { var res = dataMap.get(objectId); if (res == null) { @@ -44,28 +44,22 @@ public class FileSystemBackedStorageService implements StorageService { @Override - public void deleteObject(String objectId) { + public void deleteObject(String tenantId, String objectId) { dataMap.remove(objectId); } @Override - public boolean objectExists(String objectId) { + public boolean objectExists(String tenantId, String objectId) { return dataMap.containsKey(objectId); } - @Override - public void init() { - - } - - @Override @SneakyThrows - public void storeJSONObject(String objectId, T any) { + public void storeJSONObject(String tenantId, String objectId, T any) { File tempFile = File.createTempFile("test", ".tmp"); getMapper().writeValue(new FileOutputStream(tempFile), any); @@ -81,7 +75,7 @@ public class FileSystemBackedStorageService implements StorageService { @Override @SneakyThrows - public T readJSONObject(String objectId, Class clazz) { + public T readJSONObject(String tenantId, String objectId, Class clazz) { if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) { throw new StorageObjectDoesNotExist("Stored object not found"); @@ -104,7 +98,7 @@ public class FileSystemBackedStorageService implements StorageService { @Override @SneakyThrows - public void storeObject(String objectId, InputStream stream) { + public void storeObject(String tenantId, String objectId, InputStream stream) { File tempFile = File.createTempFile("test", ".tmp"); diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/MetricsIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/MetricsIntegrationTest.java index a2177ec..825cccf 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/MetricsIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/MetricsIntegrationTest.java @@ -22,6 +22,7 @@ import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateC import com.iqser.red.service.search.v1.server.client.IndexInformationClient; import com.iqser.red.service.search.v1.server.controller.SearchController; import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.queue.IndexingMessageReceiver; import com.iqser.red.service.search.v1.server.service.utils.MetricValidationUtils; @@ -66,7 +67,7 @@ public class MetricsIntegrationTest extends AbstractElasticsearchIntegrationTest ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - storageService.storeJSONObject(TextStorageService.StorageIdUtils.getStorageId("1", "1", FileType.SIMPLIFIED_TEXT), text); + storageService.storeJSONObject(TenantContext.getTenantId(), TextStorageService.StorageIdUtils.getStorageId("1", "1", FileType.SIMPLIFIED_TEXT), text); IndexMessage indexRequest = new IndexMessage(); indexRequest.setDossierId("1"); diff --git a/search-service-v1/search-service-server-v1/src/test/resources/application.yml b/search-service-v1/search-service-server-v1/src/test/resources/application.yml index fa75b5a..cd435bf 100644 --- a/search-service-v1/search-service-server-v1/src/test/resources/application.yml +++ b/search-service-v1/search-service-server-v1/src/test/resources/application.yml @@ -10,14 +10,7 @@ spring: allow-circular-references: true # FIXME storage: - signer-type: 'AWSS3V4SignerType' - bucket-name: 'redaction' - region: 'us-east-1' - endpoint: 'https://s3.amazonaws.com' backend: 's3' -elasticsearch: - hosts: - - 'localhost' management: endpoint: