From 8e717a506777d4cebc4fb5e0efcfa866ae87f422 Mon Sep 17 00:00:00 2001 From: Maverick Studer Date: Thu, 15 Aug 2024 09:44:49 +0200 Subject: [PATCH] RED-9872: Tenant Management issues --- .../server/queue/IndexingMessageReceiver.java | 24 ++++--- .../server/queue/MessagingConfiguration.java | 30 ++++++++ .../queue/UpdatedTenantMessageReceiver.java | 48 +++++++++++++ .../server/service/DocumentUpdateService.java | 1 + .../v1/server/service/IndexDeleteService.java | 2 + .../v1/server/service/IndexQueryResult.java | 17 +++++ .../v1/server/service/IndexQueryService.java | 9 +++ .../DocumentUpdateServiceImpl.java | 8 +++ .../elasticsearch/IndexDeleteServiceImpl.java | 11 +++ .../elasticsearch/IndexQueryServiceImpl.java | 70 +++++++++++++++++++ .../opensearch/DocumentUpdateServiceImpl.java | 8 +++ .../opensearch/IndexDeleteServiceImpl.java | 14 +++- .../opensearch/IndexQueryServiceImpl.java | 56 +++++++++++++++ 13 files changed, 288 insertions(+), 10 deletions(-) create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/UpdatedTenantMessageReceiver.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryResult.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexQueryServiceImpl.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexQueryServiceImpl.java diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index 0412798..cbea39a 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -92,10 +92,12 @@ public class IndexingMessageReceiver { log.info("Processing indexing request: {}", indexRequest); + FileModel fileStatus; + Dossier dossier; switch (indexRequest.getMessageType()) { case INSERT: - var fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); - var dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); + fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); + dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); indexFile(dossier, fileStatus); break; @@ -103,14 +105,18 @@ public class IndexingMessageReceiver { fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); - var indexUpdateDocument = indexDocumentConverterService.convertUpdateDocument(fileStatus.getAssignee(), - dossier.getSoftDeletedTime() != null, - dossier.getArchivedTime() != null, - fileStatus.getWorkflowStatus().name(), - fileStatus.getFileAttributes()); + if(documentUpdateService.documentExists(indexRequest.getFileId())) { + var indexUpdateDocument = indexDocumentConverterService.convertUpdateDocument(fileStatus.getAssignee(), + dossier.getSoftDeletedTime() != null, + dossier.getArchivedTime() != null, + fileStatus.getWorkflowStatus().name(), + fileStatus.getFileAttributes()); - documentUpdateService.updateDocument(indexRequest.getFileId(), indexUpdateDocument); - log.info("Successfully updated {}", indexRequest); + documentUpdateService.updateDocument(indexRequest.getFileId(), indexUpdateDocument); + log.info("Successfully updated {}", indexRequest); + } else { + indexFile(dossier, fileStatus); + } break; case DROP: diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java index 529b10f..a9212fb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java @@ -30,6 +30,11 @@ public class MessagingConfiguration { @Value("${fforesight.multitenancy.tenant-delete-dlq:tenant-delete-dlq}") private String tenantDeleteDLQName; + @Value("${fforesight.multitenancy.tenant-updated-queue:tenant-updated-queue}") + private String tenantUpdatedEventQueueName; + @Value("${fforesight.multitenancy.tenant-updated-dlq:tenant-updated-dlq}") + private String tenantUpdatedDLQName; + @Bean public Binding tenantExchangeDeleteBinding(@Qualifier("tenantUserManagementTenantDeleteQueue") Queue tenantUserManagementTenantDeleteQueue, @@ -39,6 +44,14 @@ public class MessagingConfiguration { } + @Bean + public Binding tenantExchangeUpdatedBinding(@Qualifier("tenantUserManagementTenantUpdatedQueue") Queue tenantUserManagementTenantUpdatedQueue, + @Qualifier("tenantExchange") TopicExchange tenantExchange) { + + return BindingBuilder.bind(tenantUserManagementTenantUpdatedQueue).to(tenantExchange).with("tenant.updated"); + } + + @Bean public Queue indexingQueue() { @@ -96,4 +109,21 @@ public class MessagingConfiguration { return QueueBuilder.durable(this.tenantDeleteDLQName).build(); } + + @Bean("tenantUserManagementTenantUpdatedQueue") + public Queue tenantUpdatedQueue() { + + return QueueBuilder.durable(this.tenantUpdatedEventQueueName) + .withArgument("x-dead-letter-exchange", "") + .withArgument("x-dead-letter-routing-key", this.tenantUpdatedDLQName) + .build(); + } + + + @Bean + public Queue tenantUpdatedDLQ() { + + return QueueBuilder.durable(this.tenantUpdatedDLQName).build(); + } + } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/UpdatedTenantMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/UpdatedTenantMessageReceiver.java new file mode 100644 index 0000000..2a24ff0 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/UpdatedTenantMessageReceiver.java @@ -0,0 +1,48 @@ +package com.iqser.red.service.search.v1.server.queue; + +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; +import com.iqser.red.service.search.v1.server.service.IndexQueryResult; +import com.iqser.red.service.search.v1.server.service.IndexQueryService; +import com.knecon.fforesight.tenantcommons.model.TenantResponse; + +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class UpdatedTenantMessageReceiver { + + private final IndexQueryService indexQueryService; + private final IndexDeleteService indexDeleteService; + + @Value("${fforesight.multitenancy.tenant-updated-queue:tenant-updated-queue}") + private String tenantUpdatedQueue; + + + @PostConstruct + public void postConstruct() { + + log.info("Listener for tenant updated events started for queue: {}", this.tenantUpdatedQueue); + } + + + @RabbitListener(queues = "${fforesight.multitenancy.tenant-updated-queue:tenant-updated-queue}") + public void updateTenant(TenantResponse tenant) { + + String numberOfReplicas = tenant.getSearchConnection().getNumberOfReplicas(); + String numberOfShards = tenant.getSearchConnection().getNumberOfShards(); + IndexQueryResult queryResult = indexQueryService.getIndexQueryResult(tenant.getSearchConnection()); + + if (queryResult.isIndexFound() && (!numberOfReplicas.equals(queryResult.getNumberOfReplicas()) || !numberOfShards.equals(queryResult.getNumberOfShards()))) { + log.info("Number of shards or replicas were changed during tenant update, indices will be recreated"); + indexDeleteService.recreateIndex(tenant.getSearchConnection()); + } + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java index 4bf1273..d83d016 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java @@ -5,5 +5,6 @@ import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; public interface DocumentUpdateService { void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate); + boolean documentExists(String fileId); } \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java index 16a8cd1..0141ed7 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java @@ -6,6 +6,8 @@ public interface IndexDeleteService { void recreateIndex(); + void recreateIndex(SearchConnection searchConnection); + void closeIndex(); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryResult.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryResult.java new file mode 100644 index 0000000..7897725 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryResult.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.search.v1.server.service; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class IndexQueryResult { + + boolean indexFound; + String numberOfShards; + String numberOfReplicas; + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryService.java new file mode 100644 index 0000000..9ccdb30 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryService.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.search.v1.server.service; + +import com.knecon.fforesight.tenantcommons.model.SearchConnection; + +public interface IndexQueryService { + + IndexQueryResult getIndexQueryResult(SearchConnection searchConnection); + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java index 9702187..0b6b4ac 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java @@ -42,4 +42,12 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService { } } + + @SneakyThrows + @Timed("redactmanager_documentExists") + public boolean documentExists(String fileId) { + + return clientCache.getClient().exists(e -> e.index(IndexNameHelper.getSearchIndex(clientCache.getClient().getSearchConnection().getIndexPrefix())).id(fileId)).value(); + } + } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java index a5e07bc..f5f4953 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java @@ -30,6 +30,17 @@ public class IndexDeleteServiceImpl implements IndexDeleteService { } + @Override + public void recreateIndex(SearchConnection searchConnection) { + + var client = new EsClient(searchConnection); + closeIndex(client, searchConnection.getIndexPrefix()); + dropIndex(client, searchConnection.getIndexPrefix()); + indexCreatorService.createIndex(client); + + } + + @SneakyThrows public void closeIndex() { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexQueryServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexQueryServiceImpl.java new file mode 100644 index 0000000..27f71f9 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexQueryServiceImpl.java @@ -0,0 +1,70 @@ +package com.iqser.red.service.search.v1.server.service.elasticsearch; + +import java.util.Optional; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.service.IndexQueryService; +import com.iqser.red.service.search.v1.server.service.IndexQueryResult; +import com.iqser.red.service.search.v1.server.utils.IndexNameHelper; +import com.knecon.fforesight.tenantcommons.model.SearchConnection; + +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.indices.GetIndicesSettingsResponse; +import co.elastic.clients.elasticsearch.indices.IndexState; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +@SuppressWarnings("PMD") +public class IndexQueryServiceImpl implements IndexQueryService { + + @SneakyThrows + public IndexQueryResult getIndexQueryResult(SearchConnection searchConnection) { + + IndexQueryResult.IndexQueryResultBuilder builder = IndexQueryResult.builder(); + + getIndexState(searchConnection).ifPresent(indexState -> { + builder.indexFound(true); + + var indexSettings = indexState.settings(); + if (indexSettings != null) { + + String replicas = indexSettings.numberOfReplicas(); + String shards = indexSettings.numberOfShards(); + + if (indexSettings.index() != null) { + + if (replicas == null) { + replicas = indexSettings.index().numberOfReplicas(); + } + if (shards == null) { + shards = indexSettings.index().numberOfShards(); + } + } + builder.numberOfReplicas(replicas).numberOfShards(shards); + } + }); + return builder.build(); + } + + + @SneakyThrows + private Optional getIndexState(SearchConnection searchConnection) { + + var esClient = new EsClient(searchConnection); + var indexName = IndexNameHelper.getSearchIndex(esClient.getSearchConnection().getIndexPrefix()); + try { + GetIndicesSettingsResponse settings = esClient.indices().getSettings(i -> i.index(indexName)); + return Optional.ofNullable(settings.get(indexName)); + } catch (ElasticsearchException elasticsearchException) { + return Optional.empty(); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java index c16c865..0716f5d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java @@ -42,4 +42,12 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService { } } + + @SneakyThrows + @Timed("redactmanager_documentExists") + public boolean documentExists(String fileId) { + + return clientCache.getClient().exists(e -> e.index(IndexNameHelper.getSearchIndex(clientCache.getClient().getSearchConnection().getIndexPrefix())).id(fileId)).value(); + } + } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java index cd3832a..0fc53d2 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java @@ -30,6 +30,17 @@ public class IndexDeleteServiceImpl implements IndexDeleteService { } + @Override + public void recreateIndex(SearchConnection searchConnection) { + + var client = new OpensearchClient(searchConnection); + closeIndex(client, searchConnection.getIndexPrefix()); + dropIndex(client, searchConnection.getIndexPrefix()); + indexCreatorService.createIndex(client); + + } + + @SneakyThrows public void closeIndex() { @@ -55,7 +66,8 @@ public class IndexDeleteServiceImpl implements IndexDeleteService { @SneakyThrows private void closeIndex(OpensearchClient opensearchClient, String indexPrefix) { - var closeIndexResponse = opensearchClient.indices().close(i -> i.index(IndexNameHelper.getSearchIndex(indexPrefix)).timeout(t -> t.time("2m"))); + var closeIndexResponse = opensearchClient.indices() + .close(i -> i.index(IndexNameHelper.getSearchIndex(indexPrefix)).timeout(t -> t.time("2m"))); if (closeIndexResponse.acknowledged()) { log.info("Index is closed"); } else { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexQueryServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexQueryServiceImpl.java new file mode 100644 index 0000000..43ee937 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexQueryServiceImpl.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import java.util.Optional; + +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch.indices.GetIndicesSettingsResponse; +import org.opensearch.client.opensearch.indices.IndexState; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.service.IndexQueryResult; +import com.iqser.red.service.search.v1.server.service.IndexQueryService; +import com.iqser.red.service.search.v1.server.utils.IndexNameHelper; +import com.knecon.fforesight.tenantcommons.model.SearchConnection; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +@SuppressWarnings("PMD") +public class IndexQueryServiceImpl implements IndexQueryService { + + @SneakyThrows + public IndexQueryResult getIndexQueryResult(SearchConnection searchConnection) { + + IndexQueryResult.IndexQueryResultBuilder builder = IndexQueryResult.builder(); + + Optional optionalIndexState = getIndexState(searchConnection); + if (optionalIndexState.isPresent()) { + builder.indexFound(true); + var indexSettings = optionalIndexState.get().settings(); + if (indexSettings != null) { + builder.numberOfReplicas(indexSettings.numberOfReplicas()).numberOfShards(indexSettings.numberOfShards()); + } + + } + return builder.build(); + } + + @SneakyThrows + private Optional getIndexState(SearchConnection searchConnection) { + + var opensearchClient = new OpensearchClient(searchConnection); + var indexName = IndexNameHelper.getSearchIndex(opensearchClient.getSearchConnection().getIndexPrefix()); + try { + GetIndicesSettingsResponse settings = opensearchClient.indices().getSettings(i -> i.index(indexName)); + return Optional.ofNullable(settings.get(indexName)); + } catch (OpenSearchException openSearchException) { + return Optional.empty(); + } + } +}