diff --git a/search-service-v1/search-service-server-v1/pom.xml b/search-service-v1/search-service-server-v1/pom.xml index c8d0ef6..10cfd0b 100644 --- a/search-service-v1/search-service-server-v1/pom.xml +++ b/search-service-v1/search-service-server-v1/pom.xml @@ -12,10 +12,15 @@ search-service-server-v1 - 1.356.0 + 1.369.0 + + com.google.guava + guava + 31.1-jre + com.iqser.red.commons storage-commons diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java index 6e0505c..9e69731 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java @@ -3,23 +3,23 @@ package com.iqser.red.service.search.v1.server; import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; -import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.search.v1.server.client.FileStatusClient; -import com.iqser.red.service.search.v1.server.service.opensearch.OpensearchClient; +import com.iqser.red.service.search.v1.server.multitenancy.AsyncConfig; +import com.iqser.red.service.search.v1.server.multitenancy.MultiTenancyMessagingConfiguration; +import com.iqser.red.service.search.v1.server.multitenancy.MultiTenancyWebConfiguration; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.instrument.MeterRegistry; -@Import({DefaultWebMvcConfiguration.class}) +@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class}) @EnableFeignClients(basePackageClasses = FileStatusClient.class) @EnableConfigurationProperties({ElasticsearchSettings.class, SearchServiceSettings.class}) @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) @@ -31,14 +31,6 @@ public class Application { } - @Bean - @ConditionalOnMissingBean - public OpensearchClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { - - return new OpensearchClient(elasticsearchSettings); - } - - @Bean public TimedAspect timedAspect(MeterRegistry registry) { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/TenantsClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/TenantsClient.java new file mode 100644 index 0000000..7c1628f --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/TenantsClient.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.search.v1.server.client; + +import org.springframework.cloud.openfeign.FeignClient; + +import com.iqser.red.service.persistence.service.v1.api.resources.TenantsResource; + +@FeignClient(name = "TenantsResource", url = "${persistence-service.url}") +public interface TenantsClient extends TenantsResource { + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java index 85d655b..9d58a48 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java @@ -8,6 +8,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.model.IndexMessage; import com.iqser.red.service.search.v1.model.IndexMessageType; +import com.iqser.red.service.search.v1.server.client.TenantsClient; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.queue.IndexingMessageReceiver; import com.iqser.red.service.search.v1.server.service.IndexInformationService; import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; @@ -24,6 +26,7 @@ public class MigrationStarterService { private final IndexInformationService indexInformationService; private final IndexingMessageReceiver indexingMessageReceiver; private final SearchServiceSettings settings; + private final TenantsClient tenantsClient; @EventListener(ApplicationReadyEvent.class) @@ -31,10 +34,13 @@ public class MigrationStarterService { // This can only run in post upgrade hook, because otherwise the old service is still runnnig. if (settings.isMigrateOnly()) { - if (indexInformationService.hasIndexChanged()) { - log.info("Index has changed and will be closed, dropped, recreated and all files will be indexed"); - indexingMessageReceiver.receiveIndexingRequest(IndexMessage.builder().messageType(IndexMessageType.DROP).build()); - } + tenantsClient.getTenants().forEach(tenant -> { + TenantContext.setTenantId(tenant.getTenantId()); + if (indexInformationService.hasIndexChanged()) { + log.info("Index has changed and will be closed, dropped, recreated and all files will be indexed"); + indexingMessageReceiver.receiveIndexingRequest(IndexMessage.builder().messageType(IndexMessageType.DROP).build()); + } + }); System.exit(SpringApplication.exit(ctx, () -> 0)); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/AsyncConfig.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/AsyncConfig.java new file mode 100644 index 0000000..12c4b72 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/AsyncConfig.java @@ -0,0 +1,27 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import java.util.concurrent.Executor; + +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.AsyncConfigurerSupport; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +@Configuration +public class AsyncConfig extends AsyncConfigurerSupport { + + @Override + public Executor getAsyncExecutor() { + + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + + executor.setCorePoolSize(7); + executor.setMaxPoolSize(42); + executor.setQueueCapacity(11); + executor.setThreadNamePrefix("TenantAwareTaskExecutor-"); + executor.setTaskDecorator(new TenantAwareTaskDecorator()); + executor.initialize(); + + return executor; + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/EncryptionDecryptionService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/EncryptionDecryptionService.java new file mode 100644 index 0000000..775eb26 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/EncryptionDecryptionService.java @@ -0,0 +1,105 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.security.spec.KeySpec; +import java.util.Base64; + +import javax.annotation.PostConstruct; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import lombok.SneakyThrows; + +@Service +public class EncryptionDecryptionService { + + @Value("${search-service.crypto.key:redaction}") + private String key; + + private SecretKey secretKey; + private byte[] iv; + + + @SneakyThrows + @PostConstruct + protected void postConstruct() { + + SecureRandom secureRandom = new SecureRandom(); + iv = new byte[12]; + secureRandom.nextBytes(iv); + secretKey = generateSecretKey(key, iv); + } + + + @SneakyThrows + public String encrypt(String strToEncrypt) { + + return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes())); + } + + + @SneakyThrows + public String decrypt(String strToDecrypt) { + + byte[] bytes = Base64.getDecoder().decode(strToDecrypt); + return new String(decrypt(bytes), StandardCharsets.UTF_8); + } + + + @SneakyThrows + public byte[] encrypt(byte[] data) { + + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec); + byte[] encryptedData = cipher.doFinal(data); + ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length); + byteBuffer.putInt(iv.length); + byteBuffer.put(iv); + byteBuffer.put(encryptedData); + return byteBuffer.array(); + } + + + @SneakyThrows + public byte[] decrypt(byte[] encryptedData) { + + ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData); + int noonceSize = byteBuffer.getInt(); + if (noonceSize < 12 || noonceSize >= 16) { + throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file."); + } + byte[] iv = new byte[noonceSize]; + byteBuffer.get(iv); + + SecretKey secretKey = generateSecretKey(key, iv); + + byte[] cipherBytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(cipherBytes); + + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); + cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec); + return cipher.doFinal(cipherBytes); + } + + + @SneakyThrows + public SecretKey generateSecretKey(String password, byte[] iv) { + + KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128 + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); + byte[] key = secretKeyFactory.generateSecret(spec).getEncoded(); + return new SecretKeySpec(key, "AES"); + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/ForwardTenantInterceptor.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/ForwardTenantInterceptor.java new file mode 100644 index 0000000..aa9255c --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/ForwardTenantInterceptor.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import org.springframework.stereotype.Component; + +import feign.RequestInterceptor; +import feign.RequestTemplate; + +@Component +public class ForwardTenantInterceptor implements RequestInterceptor { + + public static final String TENANT_HEADER_NAME = "X-TENANT-ID"; + + @Override + public void apply(RequestTemplate template) { + // do something + template.header(TENANT_HEADER_NAME, TenantContext.getTenantId()); + } +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java new file mode 100644 index 0000000..dc941e5 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/MultiTenancyMessagingConfiguration.java @@ -0,0 +1,49 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + + +import static com.iqser.red.service.search.v1.server.multitenancy.TenantInterceptor.TENANT_HEADER_NAME; + +import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class MultiTenancyMessagingConfiguration { + + @Bean + public static BeanPostProcessor multitenancyBeanPostProcessor() { + + return new BeanPostProcessor() { + + @Override + public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { + + if (bean instanceof RabbitTemplate) { + + ((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> { + m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId()); + return m; + }); + + } else if (bean instanceof AbstractRabbitListenerContainerFactory) { + + ((AbstractRabbitListenerContainerFactory) bean).setAfterReceivePostProcessors(m -> { + String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME); + + if (tenant != null) { + TenantContext.setTenantId(tenant); + } else { + throw new RuntimeException("No Tenant is set queue message"); + } + return m; + }); + } + return bean; + } + }; + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/MultiTenancyWebConfiguration.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/MultiTenancyWebConfiguration.java new file mode 100644 index 0000000..8877143 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/MultiTenancyWebConfiguration.java @@ -0,0 +1,28 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; + +import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; + +@Configuration +public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration { + + private final TenantInterceptor tenantInterceptor; + + + @Autowired + public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) { + + this.tenantInterceptor = tenantInterceptor; + } + + + @Override + public void addInterceptors(InterceptorRegistry registry) { + + registry.addWebRequestInterceptor(tenantInterceptor); + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantAwareTaskDecorator.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantAwareTaskDecorator.java new file mode 100644 index 0000000..0aa811d --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantAwareTaskDecorator.java @@ -0,0 +1,23 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import org.springframework.core.task.TaskDecorator; +import org.springframework.lang.NonNull; + +public class TenantAwareTaskDecorator implements TaskDecorator { + + @Override + @NonNull + public Runnable decorate(@NonNull Runnable runnable) { + + String tenantId = TenantContext.getTenantId(); + return () -> { + try { + TenantContext.setTenantId(tenantId); + runnable.run(); + } finally { + TenantContext.setTenantId(null); + } + }; + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantContext.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantContext.java new file mode 100644 index 0000000..e68c99f --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantContext.java @@ -0,0 +1,29 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public final class TenantContext { + + private static InheritableThreadLocal currentTenant = new InheritableThreadLocal<>(); + + + public static void setTenantId(String tenantId) { + + log.debug("Setting tenantId to " + tenantId); + currentTenant.set(tenantId); + } + + + public static String getTenantId() { + + return currentTenant.get(); + } + + + public static void clear() { + + currentTenant.remove(); + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantInterceptor.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantInterceptor.java new file mode 100644 index 0000000..0007aa1 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/multitenancy/TenantInterceptor.java @@ -0,0 +1,35 @@ +package com.iqser.red.service.search.v1.server.multitenancy; + +import org.springframework.stereotype.Component; +import org.springframework.ui.ModelMap; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.context.request.WebRequestInterceptor; + +@Component +public class TenantInterceptor implements WebRequestInterceptor { + + public static final String TENANT_HEADER_NAME = "X-TENANT-ID"; + + + @Override + public void preHandle(WebRequest request) { + + if (request.getHeader(TENANT_HEADER_NAME) != null) { + TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME)); + } + } + + + @Override + public void postHandle(WebRequest request, ModelMap model) { + + TenantContext.clear(); + } + + + @Override + public void afterCompletion(WebRequest request, Exception ex) { + + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index 99365f7..6c47563 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -27,7 +27,6 @@ import com.iqser.red.service.search.v1.server.model.Text; import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; import com.iqser.red.service.search.v1.server.service.DocumentIndexService; import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; -import com.iqser.red.service.search.v1.server.service.IndexCreatorService; import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService; import com.iqser.red.service.search.v1.server.service.IndexInformationService; @@ -51,7 +50,6 @@ public class IndexingMessageReceiver { private final DocumentDeleteService documentDeleteService; private final DocumentUpdateService documentUpdateService; - private final IndexCreatorService indexCreatorService; private final DocumentIndexService documentIndexService; private final IndexDeleteService indexDeleteService; private final IndexInformationService indexInformationService; @@ -102,9 +100,7 @@ public class IndexingMessageReceiver { break; case DROP: - indexDeleteService.closeIndex(); - indexDeleteService.dropIndex(); - indexCreatorService.createIndex(); + indexDeleteService.recreateIndex(); addAllDocumentsToIndexQueue(); try { indexInformationService.updateIndexInformation(); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java deleted file mode 100644 index 9bb4729..0000000 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.iqser.red.service.search.v1.server.service; - -public interface IndexCreatorService { - - void createIndex(); - -} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java index 9c2d21f..3a6dd16 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java @@ -2,6 +2,9 @@ package com.iqser.red.service.search.v1.server.service; public interface IndexDeleteService { + void recreateIndex(); + + void closeIndex(); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java index a10c978..dc29e58 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java @@ -1,13 +1,12 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; - import java.io.IOException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; @@ -21,16 +20,16 @@ import lombok.RequiredArgsConstructor; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") public class DocumentDeleteServiceImpl implements DocumentDeleteService { - private final EsClient client; + private final EsClientCache clientCache; private final ElasticsearchSettings settings; public void deleteDocument(String fileId) { - DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); + DeleteRequest request = new DeleteRequest.Builder().index(TenantContext.getTenantId()).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); try { - client.delete(request); + clientCache.getClient().delete(request); } catch (IOException | ElasticsearchException e) { throw IndexException.documentDeleteError(fileId, e); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java index 027cc91..4f9dd3e 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; - import java.io.IOException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -9,6 +7,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.DocumentIndexService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; @@ -24,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") public class DocumentIndexServiceImpl implements DocumentIndexService { - private final EsClient client; + private final EsClientCache clientCache; private final ElasticsearchSettings settings; @@ -32,10 +31,11 @@ public class DocumentIndexServiceImpl implements DocumentIndexService { public void indexDocument(IndexDocument indexDocument) { try { - client.index(i -> i.index(INDEX_NAME) - .id(indexDocument.getFileId()) - .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) - .document(indexDocument)); + clientCache.getClient() + .index(i -> i.index(TenantContext.getTenantId()) + .id(indexDocument.getFileId()) + .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) + .document(indexDocument)); } catch (IOException | ElasticsearchException e) { throw IndexException.documentIndexError(indexDocument.getFileId(), e); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java index 832b15e..9f050b0 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; - import java.io.IOException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -9,6 +7,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; @@ -23,7 +22,7 @@ import lombok.SneakyThrows; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") public class DocumentUpdateServiceImpl implements DocumentUpdateService { - private final EsClient client; + private final EsClientCache clientCache; private final ElasticsearchSettings settings; @@ -32,7 +31,9 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService { public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { try { - client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class); + clientCache.getClient() + .update(u -> u.index(TenantContext.getTenantId()).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), + IndexDocumentUpdate.class); } catch (IOException | ElasticsearchException e) { throw IndexException.documentUpdateError(fileId, e); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java index 0664f3d..d1fbf80 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java @@ -2,9 +2,6 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; import java.util.stream.Collectors; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; - import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; @@ -12,62 +9,49 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.persistence.service.v1.api.model.multitenancy.SearchConnection; +import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.RequiredArgsConstructor; +import lombok.Data; import lombok.experimental.Delegate; -import lombok.extern.slf4j.Slf4j; -@Slf4j -@Service -@RequiredArgsConstructor -@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +@Data public class EsClient { // Lower timeouts should be set per request. private static final int ABSURD_HIGH_TIMEOUT = 90_000_000; - private final ElasticsearchSettings settings; + private SearchConnection searchConnection; @Delegate - private co.elastic.clients.elasticsearch.ElasticsearchClient client; + private ElasticsearchClient elasticsearchClient; - @PostConstruct - public void init() { + public EsClient(SearchConnection searchConnection) { - HttpHost[] httpHost = settings.getHosts() + HttpHost[] httpHost = searchConnection.getHosts() .stream() - .map(host -> new HttpHost(host, settings.getPort(), settings.getScheme())) + .map(host -> new HttpHost(host, searchConnection.getPort(), searchConnection.getScheme())) .collect(Collectors.toList()) - .toArray(new HttpHost[settings.getHosts().size()]); + .toArray(new HttpHost[searchConnection.getHosts().size()]); RestClientBuilder builder = RestClient.builder(httpHost) .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT)); - if (settings.getUsername() != null && !settings.getUsername().isEmpty()) { + if (searchConnection.getUsername() != null && !searchConnection.getUsername().isEmpty()) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword())); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(searchConnection.getUsername(), searchConnection.getPassword())); builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } ElasticsearchTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper()); - this.client = new co.elastic.clients.elasticsearch.ElasticsearchClient(transport); - + this.searchConnection = searchConnection; + this.elasticsearchClient = new ElasticsearchClient(transport); } - - @PreDestroy - public void onShutdown() { - - client.shutdown(); - } - -} \ No newline at end of file +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java new file mode 100644 index 0000000..616a0a0 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java @@ -0,0 +1,86 @@ +package com.iqser.red.service.search.v1.server.service.elasticsearch; + +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.iqser.red.service.search.v1.server.client.TenantsClient; +import com.iqser.red.service.search.v1.server.multitenancy.EncryptionDecryptionService; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class EsClientCache { + + private final TenantsClient tenantsClient; + private final EncryptionDecryptionService encryptionDecryptionService; + private final IndexCreatorServiceImpl indexCreatorService; + + @Value("${multitenancy.client-cache.maximumSize:100}") + private Long maximumSize; + + @Value("${multitenancy.client-cache.expireAfterAccess:10}") + private Integer expireAfterAccess; + + private LoadingCache clients; + + + @PostConstruct + protected void createCache() { + + clients = CacheBuilder.newBuilder() + .maximumSize(maximumSize) + .expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES) + .removalListener((RemovalListener) removal -> { + var elasticsearchClient = removal.getValue(); + elasticsearchClient.shutdown(); + log.info("Closed elasticsearch client for tenant {}", removal.getKey()); + }) + .build(new CacheLoader<>() { + public EsClient load(String key) { + + var tenant = tenantsClient.getTenant(key); + + // Do not create new client if client with equal hosts is already available. + var hostsAsString = tenant.getSearchConnection().getHosts().stream().collect(Collectors.joining()); + for (var client : clients.asMap().values()) { + if (client.getSearchConnection().getHosts().stream().collect(Collectors.joining()).equals(hostsAsString)) { + indexCreatorService.createIndex(client); + return client; + } + } + + if (tenant.getSearchConnection().getPassword() != null) { + tenant.getSearchConnection().setPassword(encryptionDecryptionService.decrypt(tenant.getSearchConnection().getPassword())); + } + var client = new EsClient(tenant.getSearchConnection()); + log.info("Initialized elasticsearch client for tenant {}", key); + indexCreatorService.createIndex(client); + return client; + } + }); + } + + + @SneakyThrows + public EsClient getClient() { + + return clients.get(TenantContext.getTenantId()); + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java index aab4779..43d41df 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java @@ -9,52 +9,42 @@ import org.springframework.core.io.ResourceLoader; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; import co.elastic.clients.elasticsearch.indices.IndexSettings; import co.elastic.clients.elasticsearch.indices.MappingLimitSettingsNestedObjects; +import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @Service +@RequiredArgsConstructor @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") -public class IndexCreatorServiceImpl implements IndexCreatorService { +public class IndexCreatorServiceImpl { - public static final String INDEX_NAME = "redaction"; - - private final EsClient client; private final ElasticsearchSettings settings; - public IndexCreatorServiceImpl(EsClient client, ElasticsearchSettings settings) { + public void createIndex(EsClient esClient) { - this.client = client; - this.settings = settings; - - if (!indexExists()) { - createIndex(); + if (!indexExists(esClient)) { + try { + var response = esClient.indices().create(i -> i.index(TenantContext.getTenantId()).settings(createIndexSettings(esClient)).mappings(createIndexMapping())); + log.info("Successfully created index: {}", response.index()); + } catch (IOException e) { + log.error("Failed to create index.", e); + } } } - public void createIndex() { + private boolean indexExists(EsClient esClient) { try { - var response = client.indices().create(i -> i.index(INDEX_NAME).settings(createIndexSettings()).mappings(createIndexMapping())); - log.info("Successfully created index: {}", response.index()); - } catch (IOException e) { - log.error("Failed to create index.", e); - } - } - - - private boolean indexExists() { - - try { - var response = client.indices().exists(i -> i.index(INDEX_NAME)); + var response = esClient.indices().exists(i -> i.index(TenantContext.getTenantId())); return response.value(); } catch (IOException e) { throw IndexException.indexExists(e); @@ -74,14 +64,14 @@ public class IndexCreatorServiceImpl implements IndexCreatorService { @SneakyThrows - private IndexSettings createIndexSettings() { + private IndexSettings createIndexSettings(EsClient esClient) { URL resource = ResourceLoader.class.getClassLoader().getResource("index/settings.json"); try (InputStream is = resource.openStream()) { return new IndexSettings.Builder().withJson(is) - .numberOfShards(settings.getNumberOfShards()) - .numberOfReplicas(settings.getNumberOfReplicas()) + .numberOfShards(esClient.getSearchConnection().getNumberOfShards()) + .numberOfReplicas(esClient.getSearchConnection().getNumberOfReplicas()) .mapping(m -> m.nestedObjects(MappingLimitSettingsNestedObjects.of(a -> a.limit(settings.getNumberOfNestedObjectLimit())))) .build(); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java index 309311d..3a30987 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java @@ -1,11 +1,10 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; - import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import lombok.RequiredArgsConstructor; @@ -18,13 +17,22 @@ import lombok.extern.slf4j.Slf4j; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") public class IndexDeleteServiceImpl implements IndexDeleteService { - private final EsClient client; + private final EsClientCache clientCache; + private final IndexCreatorServiceImpl indexCreatorService; + + + public void recreateIndex() { + + closeIndex(); + dropIndex(); + indexCreatorService.createIndex(clientCache.getClient()); + } @SneakyThrows public void closeIndex() { - var closeIndexResponse = client.indices().close(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + var closeIndexResponse = clientCache.getClient().indices().close(i -> i.index(TenantContext.getTenantId()).timeout(t -> t.time("2m"))); if (closeIndexResponse.acknowledged()) { log.info("Index is closed"); } else { @@ -37,7 +45,7 @@ public class IndexDeleteServiceImpl implements IndexDeleteService { public void dropIndex() { log.info("Will drop index"); - var deleteIndexResponse = client.indices().delete(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + var deleteIndexResponse = clientCache.getClient().indices().delete(i -> i.index(TenantContext.getTenantId()).timeout(t -> t.time("2m"))); if (deleteIndexResponse.acknowledged()) { log.info("Index is dropped"); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java index c7f751f..347f01d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java @@ -46,7 +46,7 @@ import lombok.extern.slf4j.Slf4j; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") public class SearchServiceImpl implements SearchService { - private final EsClient client; + private final EsClientCache clientCache; @Timed("redactmanager_search") @@ -105,7 +105,7 @@ public class SearchServiceImpl implements SearchService { protected SearchResponse execute(SearchRequest searchRequest) { try { - return client.search(searchRequest, IndexDocument.class); + return clientCache.getClient().search(searchRequest, IndexDocument.class); } catch (IOException e) { throw IndexException.searchFailed(e); } @@ -306,7 +306,7 @@ public class SearchServiceImpl implements SearchService { var pages = IntStream.range(0, jsonArray.size()).mapToObj(i -> jsonArray.getInt(i)).collect(Collectors.toSet()); return MatchedSection.builder() - .headline(indexSection.get("headline") != null ? indexSection.getString("headline"): null) + .headline(indexSection.get("headline") != null ? indexSection.getString("headline") : null) .sectionNumber(indexSection.getInt("sectionNumber")) .pages(pages) .matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet())) diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java index 012eb68..78b3176 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.opensearch; -import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; - import java.io.IOException; import org.opensearch.client.opensearch._types.OpenSearchException; @@ -11,6 +9,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; @@ -21,16 +20,16 @@ import lombok.RequiredArgsConstructor; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") public class DocumentDeleteServiceImpl implements DocumentDeleteService { - private final OpensearchClient client; + private final OpensearchClientCache clientCache; private final ElasticsearchSettings settings; public void deleteDocument(String fileId) { - DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); + DeleteRequest request = new DeleteRequest.Builder().index(TenantContext.getTenantId()).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); try { - client.delete(request); + clientCache.getClient().delete(request); } catch (IOException | OpenSearchException e) { throw IndexException.documentDeleteError(fileId, e); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java index e97697b..39dbb20 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.opensearch; -import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; - import java.io.IOException; import org.opensearch.client.opensearch._types.OpenSearchException; @@ -11,6 +9,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.DocumentIndexService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; @@ -24,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") public class DocumentIndexServiceImpl implements DocumentIndexService { - private final OpensearchClient client; + private final OpensearchClientCache clientCache; private final ElasticsearchSettings settings; @@ -32,7 +31,10 @@ public class DocumentIndexServiceImpl implements DocumentIndexService { public void indexDocument(IndexDocument indexDocument) { try { - client.index(i -> i.index(INDEX_NAME).id(indexDocument.getFileId()).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).document(indexDocument)); + clientCache.getClient().index(i -> i.index(TenantContext.getTenantId()) + .id(indexDocument.getFileId()) + .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) + .document(indexDocument)); } catch (IOException | OpenSearchException e) { throw IndexException.documentIndexError(indexDocument.getFileId(), e); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java index 3861c18..f7f6fa8 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.opensearch; -import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; - import java.io.IOException; import org.opensearch.client.opensearch._types.OpenSearchException; @@ -11,6 +9,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; @@ -23,7 +22,7 @@ import lombok.SneakyThrows; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") public class DocumentUpdateServiceImpl implements DocumentUpdateService { - private final OpensearchClient client; + private final OpensearchClientCache clientCache; private final ElasticsearchSettings settings; @@ -32,8 +31,9 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService { public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { try { - client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), - IndexDocumentUpdate.class); + clientCache.getClient() + .update(u -> u.index(TenantContext.getTenantId()).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), + IndexDocumentUpdate.class); } catch (IOException | OpenSearchException e) { throw IndexException.documentUpdateError(fileId, e); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java index ba099ee..c8a5674 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java @@ -12,50 +12,41 @@ import org.springframework.core.io.ResourceLoader; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import jakarta.json.stream.JsonParser; +import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @Service +@RequiredArgsConstructor @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") -public class IndexCreatorServiceImpl implements IndexCreatorService { +public class IndexCreatorServiceImpl { - public static final String INDEX_NAME = "redaction"; - - private final OpensearchClient client; private final ElasticsearchSettings settings; - public IndexCreatorServiceImpl(OpensearchClient client, ElasticsearchSettings settings) { + public void createIndex(OpensearchClient client) { - this.client = client; - this.settings = settings; + if (!indexExists(client)) { - if (!indexExists()) { - createIndex(); + try { + var response = client.indices().create(i -> i.index(TenantContext.getTenantId()).settings(createIndexSettings(client)).mappings(createIndexMapping(client))); + log.info("Successfully created index: {}", response.index()); + } catch (IOException e) { + log.error("Failed to create index.", e); + } } } - public void createIndex() { + private boolean indexExists(OpensearchClient client) { try { - var response = client.indices().create(i -> i.index(INDEX_NAME).settings(createIndexSettings()).mappings(createIndexMapping())); - log.info("Successfully created index: {}", response.index()); - } catch (IOException e) { - log.error("Failed to create index.", e); - } - } - - - private boolean indexExists() { - - try { - var response = client.indices().exists(i -> i.index(INDEX_NAME)); + var response = client.indices().exists(i -> i.index(TenantContext.getTenantId())); return response.value(); } catch (IOException e) { throw IndexException.indexExists(e); @@ -64,7 +55,7 @@ public class IndexCreatorServiceImpl implements IndexCreatorService { @SneakyThrows - private TypeMapping createIndexMapping() { + private TypeMapping createIndexMapping(OpensearchClient client) { URL resource = ResourceLoader.class.getClassLoader().getResource("index/mapping.json"); @@ -79,7 +70,7 @@ public class IndexCreatorServiceImpl implements IndexCreatorService { @SneakyThrows - private IndexSettings createIndexSettings() { + private IndexSettings createIndexSettings(OpensearchClient client) { URL resource = ResourceLoader.class.getClassLoader().getResource("index/settings.json"); @@ -94,8 +85,8 @@ public class IndexCreatorServiceImpl implements IndexCreatorService { // Hopefully they don't hava a limit for this, I was not able to find anything. // As elasticsearch has a limit for this, and we can't set it, it seems this is the only reason for now to have both clients. var indexSettings = new IndexSettings.Builder().index(indexSettingsFromJson.index()) - .numberOfReplicas(settings.getNumberOfReplicas()) - .numberOfShards(settings.getNumberOfShards()) + .numberOfReplicas(client.getSearchConnection().getNumberOfReplicas()) + .numberOfShards(client.getSearchConnection().getNumberOfShards()) .analysis(indexSettingsFromJson.analysis()) .build(); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java index 6962bb9..26d5bc5 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java @@ -1,11 +1,10 @@ package com.iqser.red.service.search.v1.server.service.opensearch; -import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; - import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import lombok.RequiredArgsConstructor; @@ -18,13 +17,22 @@ import lombok.extern.slf4j.Slf4j; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") public class IndexDeleteServiceImpl implements IndexDeleteService { - private final OpensearchClient client; + private final OpensearchClientCache clientCache; + private final IndexCreatorServiceImpl indexCreatorService; + + + public void recreateIndex() { + + closeIndex(); + dropIndex(); + indexCreatorService.createIndex(clientCache.getClient()); + } @SneakyThrows public void closeIndex() { - var closeIndexResponse = client.indices().close(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + var closeIndexResponse = clientCache.getClient().indices().close(i -> i.index(TenantContext.getTenantId()).timeout(t -> t.time("2m"))); if (closeIndexResponse.acknowledged()) { log.info("Index is closed"); } else { @@ -37,7 +45,7 @@ public class IndexDeleteServiceImpl implements IndexDeleteService { public void dropIndex() { log.info("Will drop index"); - var deleteIndexResponse = client.indices().delete(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + var deleteIndexResponse = clientCache.getClient().indices().delete(i -> i.index(TenantContext.getTenantId()).timeout(t -> t.time("2m"))); if (deleteIndexResponse.acknowledged()) { log.info("Index is dropped"); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java index 0ae5584..cef7a9d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java @@ -2,7 +2,6 @@ package com.iqser.red.service.search.v1.server.service.opensearch; import java.util.stream.Collectors; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import org.apache.http.HttpHost; @@ -15,52 +14,45 @@ import org.opensearch.client.RestClientBuilder; import org.opensearch.client.json.jackson.JacksonJsonpMapper; import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.client.transport.rest_client.RestClientTransport; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.persistence.service.v1.api.model.multitenancy.SearchConnection; -import lombok.RequiredArgsConstructor; +import lombok.Data; import lombok.experimental.Delegate; -import lombok.extern.slf4j.Slf4j; -@Slf4j -@Service -@RequiredArgsConstructor -@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +@Data public class OpensearchClient { // Lower timeouts should be set per request. private static final int ABSURD_HIGH_TIMEOUT = 90_000_000; - private final ElasticsearchSettings settings; + private SearchConnection searchConnection; @Delegate private OpenSearchClient client; - @PostConstruct - public void init() { + public OpensearchClient(SearchConnection searchConnection) { - HttpHost[] httpHost = settings.getHosts() + HttpHost[] httpHost = searchConnection.getHosts() .stream() - .map(host -> new HttpHost(host, settings.getPort(), settings.getScheme())) + .map(host -> new HttpHost(host, searchConnection.getPort(), searchConnection.getScheme())) .collect(Collectors.toList()) - .toArray(new HttpHost[settings.getHosts().size()]); + .toArray(new HttpHost[searchConnection.getHosts().size()]); RestClientBuilder builder = RestClient.builder(httpHost) .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT)); - if (settings.getUsername() != null && !settings.getUsername().isEmpty()) { + if (searchConnection.getUsername() != null && !searchConnection.getUsername().isEmpty()) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword())); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(searchConnection.getUsername(), searchConnection.getPassword())); builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } var transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper()); + this.searchConnection = searchConnection; this.client = new OpenSearchClient(transport); - } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java new file mode 100644 index 0000000..3c1fd50 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java @@ -0,0 +1,86 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.iqser.red.service.search.v1.server.client.TenantsClient; +import com.iqser.red.service.search.v1.server.multitenancy.EncryptionDecryptionService; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class OpensearchClientCache { + + private final TenantsClient tenantsClient; + private final EncryptionDecryptionService encryptionDecryptionService; + private final IndexCreatorServiceImpl indexCreatorService; + + @Value("${multitenancy.client-cache.maximumSize:100}") + private Long maximumSize; + + @Value("${multitenancy.client-cache.expireAfterAccess:10}") + private Integer expireAfterAccess; + + private LoadingCache clients; + + + @PostConstruct + protected void createCache() { + + clients = CacheBuilder.newBuilder() + .maximumSize(maximumSize) + .expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES) + .removalListener((RemovalListener) removal -> { + var client = removal.getValue(); + client.shutdown(); + log.info("Closed opensearch client for tenant {}", removal.getKey()); + }) + .build(new CacheLoader<>() { + public OpensearchClient load(String key) { + + var tenant = tenantsClient.getTenant(key); + + // Do not create new client if client with equal hosts is already available. + var hostsAsString = tenant.getSearchConnection().getHosts().stream().collect(Collectors.joining()); + for (var client : clients.asMap().values()) { + if (client.getSearchConnection().getHosts().stream().collect(Collectors.joining()).equals(hostsAsString)) { + indexCreatorService.createIndex(client); + return client; + } + } + + if (tenant.getSearchConnection().getPassword() != null) { + tenant.getSearchConnection().setPassword(encryptionDecryptionService.decrypt(tenant.getSearchConnection().getPassword())); + } + var client = new OpensearchClient(tenant.getSearchConnection()); + log.info("Initialized elasticsearch client for tenant {}", key); + indexCreatorService.createIndex(client); + return client; + } + }); + } + + + @SneakyThrows + public OpensearchClient getClient() { + + return clients.get(TenantContext.getTenantId()); + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java index 9bb54f2..11554f1 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java @@ -47,7 +47,7 @@ import lombok.extern.slf4j.Slf4j; @ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") public class SearchServiceImpl implements SearchService { - private final OpensearchClient client; + private final OpensearchClientCache clientCache; @Timed("redactmanager_search") @@ -106,7 +106,7 @@ public class SearchServiceImpl implements SearchService { protected SearchResponse execute(SearchRequest searchRequest) { try { - return client.search(searchRequest, IndexDocument.class); + return clientCache.getClient().search(searchRequest, IndexDocument.class); } catch (IOException e) { throw IndexException.searchFailed(e); } @@ -131,9 +131,12 @@ public class SearchServiceImpl implements SearchService { var textPhraseQuery = QueryBuilders.matchPhrase().field("sections.text").query(must.toLowerCase(Locale.ROOT)).queryName(must).build()._toQuery(); var filenamePhraseQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(must.toLowerCase(Locale.ROOT)).queryName("filename." + must).build()._toQuery(); - var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value") + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase() + .field("fileAttributes.value") .query(must.toLowerCase(Locale.ROOT)) - .queryName("fileAttributes." + must).build()._toQuery(); + .queryName("fileAttributes." + must) + .build() + ._toQuery(); var filenameOrTextMustQuery = QueryBuilders.bool().should(textPhraseQuery).should(filenamePhraseQuery).should(fileAttributesPhraseQuery).build()._toQuery(); entireQuery.must(filenameOrTextMustQuery); @@ -143,9 +146,12 @@ public class SearchServiceImpl implements SearchService { var textTermQuery = QueryBuilders.matchPhrase().field("sections.text").query(should.toLowerCase(Locale.ROOT)).queryName(should).build()._toQuery(); var filenameTermQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(should.toLowerCase(Locale.ROOT)).queryName("filename." + should).build()._toQuery(); - var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value") + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase() + .field("fileAttributes.value") .query(should.toLowerCase(Locale.ROOT)) - .queryName("fileAttributes." + should).build()._toQuery(); + .queryName("fileAttributes." + should) + .build() + ._toQuery(); entireQuery.should(textTermQuery); entireQuery.should(filenameTermQuery); entireQuery.should(fileAttributesPhraseQuery); @@ -153,11 +159,14 @@ public class SearchServiceImpl implements SearchService { } if (returnSections) { - var nestedQuery = QueryBuilders.nested().scoreMode(ChildScoreMode.Avg) + var nestedQuery = QueryBuilders.nested() + .scoreMode(ChildScoreMode.Avg) .queryName("sections") .query(sectionsQueries.build()._toQuery()) .path("sections") - .innerHits(i -> i.size(100)).build()._toQuery(); + .innerHits(i -> i.size(100)) + .build() + ._toQuery(); entireQuery.should(nestedQuery); } @@ -169,7 +178,11 @@ public class SearchServiceImpl implements SearchService { for (var dossierTemplateId : dossierTemplateIds) { if (StringUtils.isNotEmpty(dossierTemplateId)) { - dossierTemplateIdQueryBuilder = dossierTemplateIdQueryBuilder.should(QueryBuilders.match().field("dossierTemplateId").query(q -> q.stringValue(dossierTemplateId)).build()._toQuery()); + dossierTemplateIdQueryBuilder = dossierTemplateIdQueryBuilder.should(QueryBuilders.match() + .field("dossierTemplateId") + .query(q -> q.stringValue(dossierTemplateId)) + .build() + ._toQuery()); } } @@ -198,15 +211,21 @@ public class SearchServiceImpl implements SearchService { } if (includeArchivedDossiers) { - filterQuery.must(QueryBuilders.terms().field("dossierArchived") - .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + filterQuery.must(QueryBuilders.terms() + .field("dossierArchived") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))) + .build() + ._toQuery()); } else { filterQuery.must(QueryBuilders.terms().field("dossierArchived").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); } if (includeDeletedDossiers) { - filterQuery.must(QueryBuilders.terms().field("dossierDeleted") - .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + filterQuery.must(QueryBuilders.terms() + .field("dossierDeleted") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))) + .build() + ._toQuery()); } else { filterQuery.must(QueryBuilders.terms().field("dossierDeleted").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); } @@ -241,7 +260,7 @@ public class SearchServiceImpl implements SearchService { return SearchResult.builder() .matchedDocuments(hits.stream().map(hit -> convertSearchHit((Hit) hit, query)).collect(Collectors.toList())) - .maxScore(response.maxScore() == null ? 0 :response.maxScore().floatValue()) + .maxScore(response.maxScore() == null ? 0 : response.maxScore().floatValue()) .total(response.hits().total().value()) .build(); } @@ -307,7 +326,7 @@ public class SearchServiceImpl implements SearchService { var pages = IntStream.range(0, jsonArray.size()).mapToObj(i -> jsonArray.getInt(i)).collect(Collectors.toSet()); return MatchedSection.builder() - .headline(indexSection.get("headline") != null ? indexSection.getString("headline"): null) + .headline(indexSection.get("headline") != null ? indexSection.getString("headline") : null) .sectionNumber(indexSection.getInt("sectionNumber")) .pages(pages) .matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet())) diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java index e3e27dc..d81b90c 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java @@ -1,8 +1,5 @@ package com.iqser.red.service.search.v1.server.settings; -import java.util.ArrayList; -import java.util.List; - import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; @@ -15,19 +12,6 @@ import lombok.Data; @ConfigurationProperties("elasticsearch") public class ElasticsearchSettings { - private List hosts = new ArrayList<>(); - - private int port = 9300; - - private String scheme = "http"; - private String apiKeyAuth; - - private String username; - - private String password; - - private String numberOfShards = "5"; - private String numberOfReplicas = "1"; private int numberOfNestedObjectLimit = 100000; /** diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index 152f75c..b0b8511 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -1,11 +1,16 @@ package com.iqser.red.service.search.v1.server.service; +import static org.mockito.Mockito.when; + +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; @@ -19,7 +24,11 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; +import com.iqser.red.service.persistence.service.v1.api.model.multitenancy.SearchConnection; +import com.iqser.red.service.persistence.service.v1.api.model.multitenancy.TenantResponse; import com.iqser.red.service.search.v1.server.Application; +import com.iqser.red.service.search.v1.server.client.TenantsClient; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @@ -34,6 +43,22 @@ public abstract class AbstractElasticsearchIntegrationTest { public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for"; public static final String SEARCH_BACKEND = "search.backend=elasticsearch"; + @MockBean + private TenantsClient tenantsClient; + + private static int port; + + + @BeforeEach + public void setupOptimize() { + + TenantContext.setTenantId("redaction"); + when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder() + .searchConnection(SearchConnection.builder().hosts(Set.of("localhost")).port(port).scheme("http").numberOfShards("1").numberOfReplicas("5").build()) + .build()); + } + + static class Initializer implements ApplicationContextInitializer { public void initialize(ConfigurableApplicationContext configurableApplicationContext) { @@ -42,10 +67,8 @@ public abstract class AbstractElasticsearchIntegrationTest { esContainer.getEnvMap().put("xpack.security.enabled", "false"); esContainer.start(); - String esHost = esContainer.getHttpHostAddress(); - int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); - - TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); + var esHost = esContainer.getHttpHostAddress(); + port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); } } diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java index f4700d8..b71966a 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java @@ -1,12 +1,17 @@ package com.iqser.red.service.search.v1.server.service; +import static org.mockito.Mockito.when; + +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.testcontainers.OpensearchContainer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; @@ -19,7 +24,11 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.testcontainers.utility.DockerImageName; +import com.iqser.red.service.persistence.service.v1.api.model.multitenancy.SearchConnection; +import com.iqser.red.service.persistence.service.v1.api.model.multitenancy.TenantResponse; import com.iqser.red.service.search.v1.server.Application; +import com.iqser.red.service.search.v1.server.client.TenantsClient; +import com.iqser.red.service.search.v1.server.multitenancy.TenantContext; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @@ -34,6 +43,22 @@ public abstract class AbstractOpensearchIntegrationTest { public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for"; public static final String SEARCH_BACKEND = "search.backend=opensearch"; + @MockBean + private TenantsClient tenantsClient; + + private static int port; + + + @BeforeEach + public void setupOptimize() { + + TenantContext.setTenantId("redaction"); + when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder() + .searchConnection(SearchConnection.builder().hosts(Set.of("localhost")).port(port).scheme("http").numberOfShards("1").numberOfReplicas("5").build()) + .build()); + } + + static class Initializer implements ApplicationContextInitializer { public void initialize(ConfigurableApplicationContext configurableApplicationContext) { @@ -43,9 +68,7 @@ public abstract class AbstractOpensearchIntegrationTest { esContainer.start(); String esHost = esContainer.getHttpHostAddress(); - int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); - - TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); + port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); } }