Merge branch 'hotfix-queue-name-mismatch' into 'master'

Fix tenant-updated queue naming mismatch

See merge request redactmanager/search-service!33

search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/MatchedSection.java

@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class MatchedSection {
 
-    private int sectionNumber;
+    private String sectionNumber;
     private String headline;
 
     @Builder.Default

search-service-v1/search-service-server-v1/build.gradle.kts

@@ -22,12 +22,13 @@ configurations {
 val springBootStarterVersion = "3.1.5"
 
 dependencies {
-    api("com.knecon.fforesight:tenant-commons:0.19.0")
-    api("com.knecon.fforesight:tracing-commons:0.3.0")
+    api("com.knecon.fforesight:tenant-commons:0.30.0")
+    api("com.knecon.fforesight:tracing-commons:0.5.0")
+    api("com.knecon.fforesight:lifecycle-commons:0.6.0")
     api("com.google.guava:guava:31.1-jre")
     api("com.iqser.red.commons:storage-commons:2.45.0")
     api(project(":search-service-api-v1"))
-    api("com.iqser.red.service:persistence-service-internal-api-v1:2.93.0")
+    api("com.iqser.red.service:persistence-service-internal-api-v1:2.576.0-RED10106.0")
     api("com.iqser.red.commons:spring-commons:2.1.0")
     api("com.iqser.red.commons:metric-commons:2.1.0")
     api("com.iqser.red.commons:jackson-commons:2.1.0")

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java

@@ -8,22 +8,25 @@ import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfi
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;
 
 import com.iqser.red.service.search.v1.server.client.FileStatusClient;
 import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
 import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, LifecycleAutoconfiguration.class})
 @Import({StorageAutoConfiguration.class})
 @EnableFeignClients(basePackageClasses = FileStatusClient.class)
 @EnableConfigurationProperties({ElasticsearchSettings.class, SearchServiceSettings.class})
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
+@EnableAspectJAutoProxy
 public class Application {
 
     public static void main(String[] args) {

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/configuration/MessagingConfiguration.java

@@ -0,0 +1,119 @@
+package com.iqser.red.service.search.v1.server.configuration;
+
+import org.springframework.amqp.core.Binding;
+import org.springframework.amqp.core.BindingBuilder;
+import org.springframework.amqp.core.DirectExchange;
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.amqp.core.TopicExchange;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import lombok.RequiredArgsConstructor;
+
+@Configuration
+@RequiredArgsConstructor
+public class MessagingConfiguration {
+
+    public static final String INDEXING_REQUEST_QUEUE_PREFIX = "indexing_request";
+    public static final String INDEXING_REQUEST_EXCHANGE = "indexing_request_exchange";
+    public static final String INDEXING_DLQ = "indexing_error";
+
+    public static final String DELETE_FROM_INDEX_REQUEST_QUEUE_PREFIX = "delete_from_index_request";
+    public static final String DELETE_FROM_INDEX_REQUEST_EXCHANGE = "delete_from_index_request_exchange";
+    public static final String DELETE_FROM_INDEX_DLQ = "delete_from_index_error";
+
+    public static final String X_ERROR_INFO_HEADER = "x-error-message";
+    public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
+
+    @Value("${fforesight.multitenancy.tenant-delete-queue:search-service-tenant-delete}")
+    private String tenantDeleteEventQueueName;
+    @Value("${fforesight.multitenancy.tenant-delete-dlq:search-service-tenant-delete-error}")
+    private String tenantDeleteDLQName;
+
+    @Value("${fforesight.multitenancy.tenant-updated-queue:search-service-tenant-updated}")
+    private String tenantUpdatedEventQueueName;
+    @Value("${fforesight.multitenancy.tenant-updated-dlq:search-service-tenant-updated-error}")
+    private String tenantUpdatedDLQName;
+
+
+    @Bean
+    public DirectExchange indexingRequestExchange() {
+
+        return new DirectExchange(INDEXING_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue indexingDLQ() {
+
+        return QueueBuilder.durable(INDEXING_DLQ).build();
+    }
+
+
+    @Bean
+    public DirectExchange deleteFromIndexRequestExchange() {
+
+        return new DirectExchange(DELETE_FROM_INDEX_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue deleteFromIndexDLQ() {
+
+        return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build();
+    }
+
+
+    @Bean
+    public Binding tenantExchangeDeleteBinding(@Qualifier("tenantUserManagementTenantDeleteQueue") Queue tenantUserManagementTenantDeleteQueue,
+                                               @Qualifier("tenantExchange") TopicExchange tenantExchange) {
+
+        return BindingBuilder.bind(tenantUserManagementTenantDeleteQueue).to(tenantExchange).with("tenant.delete");
+    }
+
+
+    @Bean("tenantUserManagementTenantDeleteQueue")
+    public Queue tenantDeleteQueue() {
+
+        return QueueBuilder.durable(this.tenantDeleteEventQueueName)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", this.tenantDeleteDLQName)
+                .build();
+    }
+
+
+    @Bean
+    public Queue tenantDeleteDLQ() {
+
+        return QueueBuilder.durable(this.tenantDeleteDLQName).build();
+    }
+
+
+    @Bean
+    public Binding tenantExchangeUpdatedBinding(@Qualifier("tenantUserManagementTenantUpdatedQueue") Queue tenantUserManagementTenantUpdatedQueue,
+                                                @Qualifier("tenantExchange") TopicExchange tenantExchange) {
+
+        return BindingBuilder.bind(tenantUserManagementTenantUpdatedQueue).to(tenantExchange).with("tenant.updated");
+    }
+
+
+    @Bean("tenantUserManagementTenantUpdatedQueue")
+    public Queue tenantUpdatedQueue() {
+
+        return QueueBuilder.durable(this.tenantUpdatedEventQueueName)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", this.tenantUpdatedDLQName)
+                .build();
+    }
+
+
+    @Bean
+    public Queue tenantUpdatedDLQ() {
+
+        return QueueBuilder.durable(this.tenantUpdatedDLQName).build();
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/configuration/TenantMessagingConfigurationImpl.java

@@ -0,0 +1,11 @@
+package com.iqser.red.service.search.v1.server.configuration;
+
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
+
+@Configuration
+public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
+
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexSection.java

@@ -16,7 +16,7 @@ import lombok.NoArgsConstructor;
 @SuppressWarnings("serial")
 public class IndexSection implements Serializable {
 
-    private int sectionNumber;
+    private String sectionNumber;
     private String text;
     private Set<Integer> pages;
     private String headline;

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/SectionText.java

@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class SectionText {
 
-    private int sectionNumber;
+    private String sectionNumber;
     private String headline;
     private String text;
 

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java

@@ -1,14 +1,12 @@
 package com.iqser.red.service.search.v1.server.queue;
 
-import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_DLQ;
-import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_QUEUE;
-import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_DQL;
-import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_QUEUE;
-import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
-import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
+import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.DELETE_FROM_INDEX_DLQ;
+import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.INDEXING_DLQ;
+import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.INDEXING_REQUEST_EXCHANGE;
+import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
+import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
@@ -20,7 +18,6 @@ import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.Dossier;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
@@ -38,6 +35,7 @@ import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
 import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService;
 import com.iqser.red.service.search.v1.server.service.IndexInformationService;
 import com.iqser.red.service.search.v1.server.service.TextStorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -48,6 +46,9 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class IndexingMessageReceiver {
 
+    public static final String INDEXING_LISTENER_ID = "indexing-listener";
+    public static final String DELETE_FROM_INDEX_LISTENER_ID = "delete-from-index-listener";
+
     private final ObjectMapper objectMapper;
     private final TextStorageService textStorageService;
     private final FileStatusClient fileStatusClient;
@@ -65,7 +66,7 @@ public class IndexingMessageReceiver {
 
     @SneakyThrows
     @RabbitHandler
-    @RabbitListener(queues = INDEXING_QUEUE)
+    @RabbitListener(id = INDEXING_LISTENER_ID)
     public void receiveIndexingRequest(Message message) {
 
         var indexRequest = objectMapper.readValue(message.getBody(), IndexMessage.class);
@@ -73,8 +74,8 @@ public class IndexingMessageReceiver {
         // This prevents from endless retries oom errors.
         if (message.getMessageProperties().isRedelivered()) {
             throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
-                    indexRequest.getDossierId(),
-                    indexRequest.getFileId()));
+                                                                      indexRequest.getDossierId(),
+                                                                      indexRequest.getFileId()));
         }
 
         try {
@@ -92,10 +93,12 @@ public class IndexingMessageReceiver {
 
         log.info("Processing indexing request: {}", indexRequest);
 
+        FileModel fileStatus;
+        Dossier dossier;
         switch (indexRequest.getMessageType()) {
             case INSERT:
-                var fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
-                var dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);
+                fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
+                dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);
                 indexFile(dossier, fileStatus);
                 break;
 
@@ -103,14 +106,17 @@ public class IndexingMessageReceiver {
                 fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
                 dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);
 
-                var indexUpdateDocument = indexDocumentConverterService.convertUpdateDocument(fileStatus.getAssignee(),
-                        dossier.getSoftDeletedTime() != null,
-                        dossier.getArchivedTime() != null,
-                        fileStatus.getWorkflowStatus().name(),
-                        fileStatus.getFileAttributes());
+                if(documentUpdateService.documentExists(indexRequest.getFileId())) {var indexUpdateDocument = indexDocumentConverterService.convertUpdateDocument(fileStatus.getAssignee(),
+                                                                                              dossier.getSoftDeletedTime() != null,
+                                                                                              dossier.getArchivedTime() != null,
+                                                                                              fileStatus.getWorkflowStatus().name(),
+                                                                                              fileStatus.getFileAttributes());
 
-                documentUpdateService.updateDocument(indexRequest.getFileId(), indexUpdateDocument);
-                log.info("Successfully updated {}", indexRequest);
+                    documentUpdateService.updateDocument(indexRequest.getFileId(), indexUpdateDocument);
+                    log.info("Successfully updated {}", indexRequest);
+                } else {
+                    indexFile(dossier, fileStatus);
+                }
                 break;
 
             case DROP:
@@ -127,7 +133,7 @@ public class IndexingMessageReceiver {
 
 
     @RabbitHandler
-    @RabbitListener(queues = INDEXING_DQL)
+    @RabbitListener(queues = INDEXING_DLQ)
     public void receiveIndexingRequestDQL(Message in) throws IOException {
 
         var indexRequest = objectMapper.readValue(in.getBody(), IndexMessage.class);
@@ -136,13 +142,15 @@ public class IndexingMessageReceiver {
         String errorMessage = errorLog + in.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
         OffsetDateTime timestamp = in.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
         timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
-        fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(), indexRequest.getFileId(), new FileErrorInfo(errorMessage, INDEXING_DQL, "search-service", timestamp));
+        fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(),
+                                                        indexRequest.getFileId(),
+                                                        new FileErrorInfo(errorMessage, INDEXING_DLQ, "search-service", timestamp));
 
     }
 
 
     @RabbitHandler
-    @RabbitListener(queues = DELETE_FROM_INDEX_QUEUE)
+    @RabbitListener(id = DELETE_FROM_INDEX_LISTENER_ID)
     public void receiveDeleteDocumentRequest(Message in) throws IOException {
 
         var indexRequest = objectMapper.readValue(in.getBody(), IndexMessage.class);
@@ -171,7 +179,9 @@ public class IndexingMessageReceiver {
         String errorMessage = errorLog + in.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
         OffsetDateTime timestamp = in.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
         timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
-        fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(), indexRequest.getFileId(), new FileErrorInfo(errorMessage, INDEXING_DQL, "search-service", timestamp));
+        fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(),
+                                                        indexRequest.getFileId(),
+                                                        new FileErrorInfo(errorMessage, INDEXING_DLQ, "search-service", timestamp));
 
     }
 
@@ -182,15 +192,15 @@ public class IndexingMessageReceiver {
         Text text = textStorageService.getText(dossier.getId(), file.getId());
 
         var indexDocument = indexDocumentConverterService.convert(dossier.getDossierTemplateId(),
-                dossier.getId(),
-                file.getId(),
-                file.getFilename(),
-                text,
-                file.getAssignee(),
-                dossier.getSoftDeletedTime() != null,
-                dossier.getArchivedTime() != null,
-                file.getWorkflowStatus(),
-                file.getFileAttributes());
+                                                                  dossier.getId(),
+                                                                  file.getId(),
+                                                                  file.getFilename(),
+                                                                  text,
+                                                                  file.getAssignee(),
+                                                                  dossier.getSoftDeletedTime() != null,
+                                                                  dossier.getArchivedTime() != null,
+                                                                  file.getWorkflowStatus(),
+                                                                  file.getFileAttributes());
 
         documentIndexService.indexDocument(indexDocument);
         fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId());
@@ -213,12 +223,13 @@ public class IndexingMessageReceiver {
 
         for (FileModel file : files) {
             log.info("Will add dossier {} file {} to index queue", dossierId, file.getId());
-            rabbitTemplate.convertAndSend(INDEXING_QUEUE,
-                    IndexMessage.builder().messageType(IndexMessageType.INSERT).dossierId(dossierId).fileId(file.getId()).build(),
-                    message -> {
-                        message.getMessageProperties().setPriority(99);
-                        return message;
-                    });
+            rabbitTemplate.convertAndSend(INDEXING_REQUEST_EXCHANGE,
+                                          TenantContext.getTenantId(),
+                                          IndexMessage.builder().messageType(IndexMessageType.INSERT).dossierId(dossierId).fileId(file.getId()).build(),
+                                          message -> {
+                                              message.getMessageProperties().setPriority(99);
+                                              return message;
+                                          });
         }
     }
 

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java

@@ -1,99 +0,0 @@
-package com.iqser.red.service.search.v1.server.queue;
-
-import org.springframework.amqp.core.Binding;
-import org.springframework.amqp.core.BindingBuilder;
-import org.springframework.amqp.core.Queue;
-import org.springframework.amqp.core.QueueBuilder;
-import org.springframework.amqp.core.TopicExchange;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-import lombok.RequiredArgsConstructor;
-
-@Configuration
-@RequiredArgsConstructor
-public class MessagingConfiguration {
-
-    public static final String INDEXING_QUEUE = "indexingQueue";
-    public static final String INDEXING_DQL = "indexingDQL";
-
-    public static final String DELETE_FROM_INDEX_QUEUE = "deleteFromIndexQueue";
-    public static final String DELETE_FROM_INDEX_DLQ = "deleteFromIndexDLQ";
-
-    public static final String X_ERROR_INFO_HEADER = "x-error-message";
-    public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
-
-    @Value("${fforesight.multitenancy.tenant-delete-queue:tenant-delete-queue}")
-    private String tenantDeleteEventQueueName;
-    @Value("${fforesight.multitenancy.tenant-delete-dlq:tenant-delete-dlq}")
-    private String tenantDeleteDLQName;
-
-
-    @Bean
-    public Binding tenantExchangeDeleteBinding(@Qualifier("tenantUserManagementTenantDeleteQueue") Queue tenantUserManagementTenantDeleteQueue,
-                                               @Qualifier("tenantExchange") TopicExchange tenantExchange) {
-
-        return BindingBuilder.bind(tenantUserManagementTenantDeleteQueue).to(tenantExchange).with("tenant.delete");
-    }
-
-
-    @Bean
-    public Queue indexingQueue() {
-
-        return QueueBuilder.durable(INDEXING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", INDEXING_DQL).maxPriority(2).build();
-    }
-
-
-    @Bean
-    public Queue indexingDeadLetterQueue() {
-
-        return QueueBuilder.durable(INDEXING_DQL).build();
-    }
-
-
-    @Bean
-    public Queue deleteFromIndexQueue() {
-
-        return QueueBuilder.durable(DELETE_FROM_INDEX_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", DELETE_FROM_INDEX_DLQ)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue deleteFromIndexDLQ() {
-
-        return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build();
-    }
-
-    // Tentant Delete Event Queue
-
-
-    @Bean(name = "tenantExchange")
-    TopicExchange tenantExchange(@Value("${fforesight.tenant-exchange.name}") String tenantExchangeName) {
-
-        return new TopicExchange(tenantExchangeName);
-    }
-
-
-    @Bean("tenantUserManagementTenantDeleteQueue")
-    public Queue tenantDeleteQueue() {
-
-        return QueueBuilder.durable(this.tenantDeleteEventQueueName)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", this.tenantDeleteDLQName)
-                .build();
-    }
-
-
-    @Bean
-    public Queue tenantDeleteDLQ() {
-
-        return QueueBuilder.durable(this.tenantDeleteDLQName).build();
-    }
-
-}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/TenantExchangeMessageReceiverImpl.java

@@ -0,0 +1,74 @@
+package com.iqser.red.service.search.v1.server.queue;
+
+import static com.iqser.red.service.search.v1.server.configuration.MessagingConfiguration.*;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.tenantcommons.TenantProvider;
+import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
+import com.knecon.fforesight.tenantcommons.model.TenantResponse;
+import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
+import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
+
+@Service
+public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
+
+    public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
+
+        super(rabbitQueueService, tenantProvider);
+    }
+
+
+    @Override
+    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
+
+        return Set.of(TenantQueueConfiguration.builder()
+                              .listenerId(IndexingMessageReceiver.INDEXING_LISTENER_ID)
+                              .exchangeName(INDEXING_REQUEST_EXCHANGE)
+                              .queuePrefix(INDEXING_REQUEST_QUEUE_PREFIX)
+                              .dlqName(INDEXING_DLQ)
+                              .arguments(Map.of("x-max-priority", 2))
+                              .build(),
+                      TenantQueueConfiguration.builder()
+                              .listenerId(IndexingMessageReceiver.DELETE_FROM_INDEX_LISTENER_ID)
+                              .exchangeName(DELETE_FROM_INDEX_REQUEST_EXCHANGE)
+                              .queuePrefix(DELETE_FROM_INDEX_REQUEST_QUEUE_PREFIX)
+                              .dlqName(DELETE_FROM_INDEX_DLQ)
+                              .arguments(Map.of("x-max-priority", 2))
+                              .build());
+    }
+
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void onApplicationReady() {
+
+        System.out.println("application ready invoked");
+        super.initializeQueues();
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
+    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
+
+        super.reactToTenantCreation(tenantCreatedEvent);
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
+    public void reactToTenantDeletion(TenantResponse tenantResponse) {
+
+        super.reactToTenantDeletion(tenantResponse);
+
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/UpdatedTenantMessageReceiver.java

@@ -0,0 +1,48 @@
+package com.iqser.red.service.search.v1.server.queue;
+
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
+import com.iqser.red.service.search.v1.server.service.IndexQueryResult;
+import com.iqser.red.service.search.v1.server.service.IndexQueryService;
+import com.knecon.fforesight.tenantcommons.model.TenantResponse;
+
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class UpdatedTenantMessageReceiver {
+
+    private final IndexQueryService indexQueryService;
+    private final IndexDeleteService indexDeleteService;
+
+    @Value("${fforesight.multitenancy.tenant-updated-queue:search-service-tenant-updated}")
+    private String tenantUpdatedQueue;
+
+
+    @PostConstruct
+    public void postConstruct() {
+
+        log.info("Listener for tenant updated events started for queue: {}", this.tenantUpdatedQueue);
+    }
+
+
+    @RabbitListener(queues = "${fforesight.multitenancy.tenant-updated-queue:search-service-tenant-updated}")
+    public void updateTenant(TenantResponse tenant) {
+
+        String numberOfReplicas = tenant.getSearchConnection().getNumberOfReplicas();
+        String numberOfShards = tenant.getSearchConnection().getNumberOfShards();
+        IndexQueryResult queryResult = indexQueryService.getIndexQueryResult(tenant.getSearchConnection());
+
+        if (queryResult.isIndexFound() && (!numberOfReplicas.equals(queryResult.getNumberOfReplicas()) || !numberOfShards.equals(queryResult.getNumberOfShards()))) {
+            log.info("Number of shards or replicas were changed during tenant update, indices will be recreated");
+            indexDeleteService.recreateIndex(tenant.getSearchConnection());
+        }
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java

@@ -5,5 +5,6 @@ import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate;
 public interface DocumentUpdateService {
 
     void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate);
+    boolean documentExists(String fileId);
 
 }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java

@@ -6,6 +6,8 @@ public interface IndexDeleteService {
 
     void recreateIndex();
 
+    void recreateIndex(SearchConnection searchConnection);
+
 
     void closeIndex();
 

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryResult.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.search.v1.server.service;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class IndexQueryResult {
+
+    boolean indexFound;
+    String numberOfShards;
+    String numberOfReplicas;
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexQueryService.java

@@ -0,0 +1,9 @@
+package com.iqser.red.service.search.v1.server.service;
+
+import com.knecon.fforesight.tenantcommons.model.SearchConnection;
+
+public interface IndexQueryService {
+
+    IndexQueryResult getIndexQueryResult(SearchConnection searchConnection);
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java

@@ -34,6 +34,7 @@ public class DocumentDeleteServiceImpl implements DocumentDeleteService {
         try {
             clientCache.getClient().delete(request);
         } catch (IOException | ElasticsearchException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.documentDeleteError(fileId, e);
         }
     }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java

@@ -37,6 +37,7 @@ public class DocumentIndexServiceImpl implements DocumentIndexService {
                             .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy()))
                             .document(indexDocument));
         } catch (IOException | ElasticsearchException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.documentIndexError(indexDocument.getFileId(), e);
         }
     }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java

@@ -37,8 +37,17 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService {
                             .doc(indexDocumentUpdate)
                             .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class);
         } catch (IOException | ElasticsearchException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.documentUpdateError(fileId, e);
         }
     }
 
+
+    @SneakyThrows
+    @Timed("redactmanager_documentExists")
+    public boolean documentExists(String fileId) {
+
+        return clientCache.getClient().exists(e -> e.index(IndexNameHelper.getSearchIndex(clientCache.getClient().getSearchConnection().getIndexPrefix())).id(fileId)).value();
+    }
+
 }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch;
 
 import java.util.stream.Collectors;
 
+import lombok.SneakyThrows;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
@@ -24,7 +25,7 @@ import lombok.experimental.Delegate;
 public class EsClient {
 
     // Lower timeouts should be set per request.
-    private static final int ABSURD_HIGH_TIMEOUT = 90_000_000;
+    private static final int ABSURD_HIGH_TIMEOUT = 600_000;
 
     private SearchConnection searchConnection;
 
@@ -37,11 +38,12 @@ public class EsClient {
         HttpHost[] httpHost = searchConnection.getHosts()
                 .stream()
                 .map(host -> new HttpHost(host, searchConnection.getPort(), searchConnection.getScheme()))
-                .collect(Collectors.toList())
+                .toList()
                 .toArray(new HttpHost[searchConnection.getHosts().size()]);
 
-        RestClientBuilder builder = RestClient.builder(httpHost)
-                .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT));
+        var builder = RestClient.builder(httpHost)
+                .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT)
+                        .setSocketTimeout(ABSURD_HIGH_TIMEOUT));
 
         if (searchConnection.getUsername() != null && !searchConnection.getUsername().isEmpty()) {
             final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -55,4 +57,10 @@ public class EsClient {
         this.elasticsearchClient = new ElasticsearchClient(transport);
     }
 
+    @SneakyThrows
+    public void terminate() {
+
+        elasticsearchClient._transport().close();
+    }
+
 }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClientCache.java

@@ -38,6 +38,30 @@ public class EsClientCache {
     private LoadingCache<String, EsClient> clients;
 
 
+    @SneakyThrows
+    public void isClientAliveOrTerminate() {
+
+        try {
+            var client = clients.get(TenantContext.getTenantId());
+            try {
+
+                log.info("Checking if client is still alive: {}", client.info());
+            } catch (Exception e) {
+
+                try {
+                    client.terminate();
+                } catch (Exception e2) {
+
+                    log.info("Failed to terminate ES Client");
+                    clients.invalidate(TenantContext.getTenantId());
+                }
+            }
+        }catch (Exception e){
+            log.error("Failed to terminate/invalide client", e);
+        }
+    }
+
+
     @PostConstruct
     protected void createCache() {
 
@@ -45,8 +69,12 @@ public class EsClientCache {
                 .maximumSize(maximumSize)
                 .expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES)
                 .removalListener((RemovalListener<String, EsClient>) removal -> {
-                    removal.getValue().shutdown();
-                    log.info("Closed elasticsearch client for tenant {}", removal.getKey());
+                    try {
+                        removal.getValue().terminate();
+                        log.info("Closed elasticsearch client for tenant {}", removal.getKey());
+                    } catch (Exception e) {
+                        log.info("Failed to close elasticsearch client for tenant {}", removal.getKey());
+                    }
                 })
                 .build(new CacheLoader<>() {
                     public EsClient load(String tenantId) {

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java

@@ -30,6 +30,17 @@ public class IndexDeleteServiceImpl implements IndexDeleteService {
     }
 
 
+    @Override
+    public void recreateIndex(SearchConnection searchConnection) {
+
+        var client = new EsClient(searchConnection);
+        closeIndex(client, searchConnection.getIndexPrefix());
+        dropIndex(client, searchConnection.getIndexPrefix());
+        indexCreatorService.createIndex(client);
+
+    }
+
+
     @SneakyThrows
     public void closeIndex() {
 

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexQueryServiceImpl.java

@@ -0,0 +1,70 @@
+package com.iqser.red.service.search.v1.server.service.elasticsearch;
+
+import java.util.Optional;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.search.v1.server.service.IndexQueryService;
+import com.iqser.red.service.search.v1.server.service.IndexQueryResult;
+import com.iqser.red.service.search.v1.server.utils.IndexNameHelper;
+import com.knecon.fforesight.tenantcommons.model.SearchConnection;
+
+import co.elastic.clients.elasticsearch._types.ElasticsearchException;
+import co.elastic.clients.elasticsearch.indices.GetIndicesSettingsResponse;
+import co.elastic.clients.elasticsearch.indices.IndexState;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
+@SuppressWarnings("PMD")
+public class IndexQueryServiceImpl implements IndexQueryService {
+
+    @SneakyThrows
+    public IndexQueryResult getIndexQueryResult(SearchConnection searchConnection) {
+
+        IndexQueryResult.IndexQueryResultBuilder builder = IndexQueryResult.builder();
+
+        getIndexState(searchConnection).ifPresent(indexState -> {
+            builder.indexFound(true);
+
+            var indexSettings = indexState.settings();
+            if (indexSettings != null) {
+
+                String replicas = indexSettings.numberOfReplicas();
+                String shards = indexSettings.numberOfShards();
+
+                if (indexSettings.index() != null) {
+
+                    if (replicas == null) {
+                        replicas = indexSettings.index().numberOfReplicas();
+                    }
+                    if (shards == null) {
+                        shards = indexSettings.index().numberOfShards();
+                    }
+                }
+                builder.numberOfReplicas(replicas).numberOfShards(shards);
+            }
+        });
+        return builder.build();
+    }
+
+
+    @SneakyThrows
+    private Optional<IndexState> getIndexState(SearchConnection searchConnection) {
+
+        var esClient = new EsClient(searchConnection);
+        var indexName = IndexNameHelper.getSearchIndex(esClient.getSearchConnection().getIndexPrefix());
+        try {
+            GetIndicesSettingsResponse settings = esClient.indices().getSettings(i -> i.index(indexName));
+            return Optional.ofNullable(settings.get(indexName));
+        } catch (ElasticsearchException elasticsearchException) {
+            return Optional.empty();
+        }
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java

@@ -109,6 +109,7 @@ public class SearchServiceImpl implements SearchService {
         try {
             return clientCache.getClient().search(searchRequest, IndexDocument.class);
         } catch (IOException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.searchFailed(e);
         }
     }
@@ -309,7 +310,7 @@ public class SearchServiceImpl implements SearchService {
 
         return MatchedSection.builder()
                 .headline(indexSection.get("headline") != null ? indexSection.getString("headline") : null)
-                .sectionNumber(indexSection.getInt("sectionNumber"))
+                .sectionNumber(indexSection.getString("sectionNumber"))
                 .pages(pages)
                 .matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet()))
                 .build();

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java

@@ -34,6 +34,7 @@ public class DocumentDeleteServiceImpl implements DocumentDeleteService {
         try {
             clientCache.getClient().delete(request);
         } catch (IOException | OpenSearchException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.documentDeleteError(fileId, e);
         }
     }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java

@@ -36,6 +36,7 @@ public class DocumentIndexServiceImpl implements DocumentIndexService {
                     .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy()))
                     .document(indexDocument));
         } catch (IOException | OpenSearchException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.documentIndexError(indexDocument.getFileId(), e);
         }
     }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java

@@ -37,8 +37,17 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService {
                             .doc(indexDocumentUpdate)
                             .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class);
         } catch (IOException | OpenSearchException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.documentUpdateError(fileId, e);
         }
     }
 
+
+    @SneakyThrows
+    @Timed("redactmanager_documentExists")
+    public boolean documentExists(String fileId) {
+
+        return clientCache.getClient().exists(e -> e.index(IndexNameHelper.getSearchIndex(clientCache.getClient().getSearchConnection().getIndexPrefix())).id(fileId)).value();
+    }
+
 }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java

@@ -30,6 +30,17 @@ public class IndexDeleteServiceImpl implements IndexDeleteService {
     }
 
 
+    @Override
+    public void recreateIndex(SearchConnection searchConnection) {
+
+        var client = new OpensearchClient(searchConnection);
+        closeIndex(client, searchConnection.getIndexPrefix());
+        dropIndex(client, searchConnection.getIndexPrefix());
+        indexCreatorService.createIndex(client);
+
+    }
+
+
     @SneakyThrows
     public void closeIndex() {
 
@@ -55,7 +66,8 @@ public class IndexDeleteServiceImpl implements IndexDeleteService {
     @SneakyThrows
     private void closeIndex(OpensearchClient opensearchClient, String indexPrefix) {
 
-        var closeIndexResponse = opensearchClient.indices().close(i -> i.index(IndexNameHelper.getSearchIndex(indexPrefix)).timeout(t -> t.time("2m")));
+        var closeIndexResponse = opensearchClient.indices()
+                .close(i -> i.index(IndexNameHelper.getSearchIndex(indexPrefix)).timeout(t -> t.time("2m")));
         if (closeIndexResponse.acknowledged()) {
             log.info("Index is closed");
         } else {

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexQueryServiceImpl.java

@@ -0,0 +1,56 @@
+package com.iqser.red.service.search.v1.server.service.opensearch;
+
+import java.util.Optional;
+
+import org.opensearch.client.opensearch._types.OpenSearchException;
+import org.opensearch.client.opensearch.indices.GetIndicesSettingsResponse;
+import org.opensearch.client.opensearch.indices.IndexState;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.search.v1.server.service.IndexQueryResult;
+import com.iqser.red.service.search.v1.server.service.IndexQueryService;
+import com.iqser.red.service.search.v1.server.utils.IndexNameHelper;
+import com.knecon.fforesight.tenantcommons.model.SearchConnection;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
+@SuppressWarnings("PMD")
+public class IndexQueryServiceImpl implements IndexQueryService {
+
+    @SneakyThrows
+    public IndexQueryResult getIndexQueryResult(SearchConnection searchConnection) {
+
+        IndexQueryResult.IndexQueryResultBuilder builder = IndexQueryResult.builder();
+
+        Optional<IndexState> optionalIndexState = getIndexState(searchConnection);
+        if (optionalIndexState.isPresent()) {
+            builder.indexFound(true);
+            var indexSettings = optionalIndexState.get().settings();
+            if (indexSettings != null) {
+                builder.numberOfReplicas(indexSettings.numberOfReplicas()).numberOfShards(indexSettings.numberOfShards());
+            }
+
+        }
+        return builder.build();
+    }
+
+    @SneakyThrows
+    private Optional<IndexState> getIndexState(SearchConnection searchConnection) {
+
+        var opensearchClient = new OpensearchClient(searchConnection);
+        var indexName = IndexNameHelper.getSearchIndex(opensearchClient.getSearchConnection().getIndexPrefix());
+        try {
+            GetIndicesSettingsResponse settings = opensearchClient.indices().getSettings(i -> i.index(indexName));
+            return Optional.ofNullable(settings.get(indexName));
+        } catch (OpenSearchException openSearchException) {
+            return Optional.empty();
+        }
+    }
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java

@@ -1,30 +1,25 @@
 package com.iqser.red.service.search.v1.server.service.opensearch;
 
-import java.util.stream.Collectors;
-
+import com.knecon.fforesight.tenantcommons.model.SearchConnection;
+import lombok.Data;
+import lombok.SneakyThrows;
+import lombok.experimental.Delegate;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.opensearch.client.RestClient;
-import org.opensearch.client.RestClientBuilder;
 import org.opensearch.client.json.jackson.JacksonJsonpMapper;
 import org.opensearch.client.opensearch.OpenSearchClient;
 import org.opensearch.client.transport.rest_client.RestClientTransport;
 
-import com.knecon.fforesight.tenantcommons.model.SearchConnection;
-
-import jakarta.annotation.PreDestroy;
-import lombok.Data;
-import lombok.experimental.Delegate;
-
 @Data
 @SuppressWarnings("PMD")
 public class OpensearchClient {
 
     // Lower timeouts should be set per request.
-    private static final int ABSURD_HIGH_TIMEOUT = 90_000_000;
+    private static final int ABSURD_HIGH_TIMEOUT = 600_000;
 
     private SearchConnection searchConnection;
 
@@ -37,11 +32,13 @@ public class OpensearchClient {
         HttpHost[] httpHost = searchConnection.getHosts()
                 .stream()
                 .map(host -> new HttpHost(host, searchConnection.getPort(), searchConnection.getScheme()))
-                .collect(Collectors.toList())
+                .toList()
                 .toArray(new HttpHost[searchConnection.getHosts().size()]);
 
-        RestClientBuilder builder = RestClient.builder(httpHost)
-                .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT));
+        var builder = RestClient.builder(httpHost)
+                .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(0)
+                        .setConnectionRequestTimeout(ABSURD_HIGH_TIMEOUT)
+                        .setSocketTimeout(ABSURD_HIGH_TIMEOUT));
 
         if (searchConnection.getUsername() != null && !searchConnection.getUsername().isEmpty()) {
             final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
@@ -56,10 +53,10 @@ public class OpensearchClient {
     }
 
 
-    @PreDestroy
-    public void onShutdown() {
+    @SneakyThrows
+    public void terminate() {
 
-        client.shutdown();
+        client._transport().close();
     }
 
 }

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClientCache.java

@@ -38,6 +38,29 @@ public class OpensearchClientCache {
     private LoadingCache<String, OpensearchClient> clients;
 
 
+    @SneakyThrows
+    public void isClientAliveOrTerminate() {
+
+        try {
+            var client = clients.get(TenantContext.getTenantId());
+            try {
+
+                log.info("Checking if client is still alive: {}", client.info());
+            } catch (Exception e) {
+
+                try {
+                    client.terminate();
+                } catch (Exception e2) {
+
+                    log.info("Failed to terminate ES Client");
+                    clients.invalidate(TenantContext.getTenantId());
+                }
+            }
+        }catch (Exception e){
+            log.error("Failed to terminate/invalide client", e);
+        }
+    }
+
     @PostConstruct
     protected void createCache() {
 
@@ -45,8 +68,12 @@ public class OpensearchClientCache {
                 .maximumSize(maximumSize)
                 .expireAfterAccess(expireAfterAccess, TimeUnit.MINUTES)
                 .removalListener((RemovalListener<String, OpensearchClient>) removal -> {
-                    removal.getValue().shutdown();
-                    log.info("Closed elasticsearch client for tenant {}", removal.getKey());
+                    try {
+                        removal.getValue().terminate();
+                        log.info("Closed elasticsearch client for tenant {}", removal.getKey());
+                    } catch (Exception e) {
+                        log.info("Failed to close elasticsearch client for tenant {}", removal.getKey());
+                    }
                 })
                 .build(new CacheLoader<>() {
                     public OpensearchClient load(String tenantId) {

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java

@@ -110,6 +110,7 @@ public class SearchServiceImpl implements SearchService {
         try {
             return clientCache.getClient().search(searchRequest, IndexDocument.class);
         } catch (IOException e) {
+            clientCache.isClientAliveOrTerminate();
             throw IndexException.searchFailed(e);
         }
     }
@@ -329,7 +330,7 @@ public class SearchServiceImpl implements SearchService {
 
         return MatchedSection.builder()
                 .headline(indexSection.get("headline") != null ? indexSection.getString("headline") : null)
-                .sectionNumber(indexSection.getInt("sectionNumber"))
+                .sectionNumber(indexSection.getString("sectionNumber"))
                 .pages(pages)
                 .matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet()))
                 .build();

search-service-v1/search-service-server-v1/src/main/resources/application.yml

@@ -18,6 +18,9 @@ project.version: 1.0-SNAPSHOT
 server:
   port: 8080
 
+lifecycle:
+  base-package: com.iqser.red.service.search
+
 spring:
   application:
     name: search-service

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java

@@ -6,6 +6,8 @@ import java.util.Set;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -52,6 +54,12 @@ public abstract class AbstractElasticsearchIntegrationTest {
     @MockBean
     private TenantsClient tenantsClient;
 
+    @MockBean
+    private RabbitAdmin rabbitAdmin;
+
+    @MockBean
+    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
+
     private static int port;
 
 
@@ -60,15 +68,15 @@ public abstract class AbstractElasticsearchIntegrationTest {
 
         TenantContext.setTenantId("redaction");
         when(tenantsClient.getTenant("redaction")).thenReturn(TenantResponse.builder()
-                .searchConnection(SearchConnection.builder()
-                        .hosts(Set.of("localhost"))
-                        .port(port)
-                        .scheme("http")
-                        .numberOfShards("1")
-                        .numberOfReplicas("5")
-                        .indexPrefix("indexprefix")
-                        .build())
-                .build());
+                                                                      .searchConnection(SearchConnection.builder()
+                                                                                                .hosts(Set.of("localhost"))
+                                                                                                .port(port)
+                                                                                                .scheme("http")
+                                                                                                .numberOfShards("1")
+                                                                                                .numberOfReplicas("5")
+                                                                                                .indexPrefix("indexprefix")
+                                                                                                .build())
+                                                                      .build());
     }
 
 

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java

@@ -7,7 +7,9 @@ import java.util.List;
 import java.util.Map;
 
 import org.junit.jupiter.api.Test;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.core.io.ClassPathResource;

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/MetricsIntegrationTest.java

@@ -3,7 +3,9 @@ package com.iqser.red.service.search.v1.server.service;
 import static org.mockito.Mockito.when;
 
 import org.junit.jupiter.api.Test;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.core.io.ClassPathResource;

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java

@@ -7,7 +7,9 @@ import java.util.List;
 import java.util.Map;
 
 import org.junit.jupiter.api.Test;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.core.io.ClassPathResource;
@@ -54,6 +56,12 @@ public class OpensearchTest extends AbstractOpensearchIntegrationTest {
     @MockBean
     private RabbitTemplate rabbitTemplate;
 
+    @MockBean
+    private RabbitAdmin rabbitAdmin;
+
+    @MockBean
+    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
+
     @MockBean
     private IndexDeleteService indexDeleteService;
 

search-service-v1/search-service-server-v1/src/test/resources/application.yml

@@ -32,3 +32,5 @@ persistence-service.url: 'http://mock.url'
 
 server:
   port: 19547
+
+POD_NAME: search-service

search-service-v1/search-service-server-v1/src/test/resources/files/crafted document.json

@@ -2,204 +2,204 @@
   "numberOfPages": 9,
   "sectionTexts": [
     {
-      "sectionNumber": 1,
+      "sectionNumber": "1",
       "text": "Rule 0: Expand CBI Authors with firstname initials F. Lastname, J. Doe, M. Mustermann Lastname M., Doe J. Mustermann M."
     },
     {
-      "sectionNumber": 2,
+      "sectionNumber": "2",
       "text": "Rule 1/2: Redact CBI Authors based on Dict Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No David Ksenia Max Mustermann Ranya Eikenboom Charalampos Schenk Tanja Schmitt ← should not be annotated, not in Dictionary"
     },
     {
-      "sectionNumber": 3,
+      "sectionNumber": "3",
       "text": "Rule 3/4: Redact (not) CBI Add/ress based on Dict Dont Redact (mark as skipped) when Vertebrate Study is No Redact when Vertebrate Study is Yes Warnsveld, 7232 CX Warnsveld, Netherlands, NL Institut Industries, 33 Rue Jean Baffier, 18000 Bourges, France, FR 4-6 Chem. des Varennes, 18300 Saint-Satur, France, FR Lesdo Industries, Chäppelisträssli, 6078 Lungern, Switzerland Shlissel'burgskaya Ulitsa, Nizhny Novgorod Oblast, Russia, 603034, RU Karl Johans Gate 11, 0154 Oslo, Norway, NOR ← should not be annotated, not in Dictionary"
     },
     {
-      "sectionNumber": 4,
+      "sectionNumber": "4",
       "text": "Rule 5: Do not redact genitive CBI_authors (Entries based on Dict) Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated"
     },
     {
-      "sectionNumber": 5,
+      "sectionNumber": "5",
       "text": "Reference No Author(s) Year Title Laboratory"
     },
     {
-      "sectionNumber": 6,
+      "sectionNumber": "6",
       "text": "BR2 /2 Michael N. 1998 The role of physical education in the school system. Weyland Industries"
     },
     {
-      "sectionNumber": 7,
+      "sectionNumber": "7",
       "text": "BR3 /5 Funnarie B. 2001 It should be illegal to produce and sell tobacco Authentic Diagnostics"
     },
     {
-      "sectionNumber": 8,
+      "sectionNumber": "8",
       "text": "ZZ/12 Feuer A. 1989 Social media is the real cause of teenage depression. Tyrell Corporation"
     },
     {
-      "sectionNumber": 10,
+      "sectionNumber": "10",
       "text": "Rule 6-11 (Authors Table) Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No"
     },
     {
-      "sectionNumber": 11,
+      "sectionNumber": "11",
       "text": "Rule 12/13: Redact/Hint if CTL or BL was found Redact when Vertebrate Study is Yes Hint when Vertebrate Study is No CTL/without dictionary entry CTL without Slash BL/without dictionary entry BL without Slash CTL/with dictionary entry 1234 with Slash CTL with dictionary entry 5678 without Slash BL/with dictionary entry 1234 with Slash BL with dictionary entry 5678 without Slash"
     },
     {
-      "sectionNumber": 12,
+      "sectionNumber": "12",
       "text": "Rule 14/15: Redact and add recommendation for et al. Redact Term “Desiree”, “Melanie” and add to Recommendation CBI Authors if Vertebrate Study is Yes & No Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum."
     },
     {
-      "sectionNumber": 13,
+      "sectionNumber": "13",
       "text": "Rule 16/17: Add recommendation for Addresses in Test Organism/Animals sections Recommend only if Vertebrate Study is Yes, else do nothing Lorem ipsum dolor sit Species: Mouse; Source: Stark Industries"
     },
     {
-      "sectionNumber": 14,
+      "sectionNumber": "14",
       "text": "Rule 16/17 (additional) negative Test; missing first Key Nothing should happen because of missing first/second keyword according to the rules Dont redact here because of missing first key; Source: Omni Consumer Products Dont redact here because missing first keyword; Source Resources Development Administration"
     },
     {
-      "sectionNumber": 15,
+      "sectionNumber": "15",
       "text": "Rule 16/17 (additional) negative Test; missing second Key Dont redact here because of missing second key; Species: Mouse; Omni Consumer Products Dont redact here because missing second keyword; Species: Mouse, Resources Development Administration"
     },
     {
-      "sectionNumber": 16,
+      "sectionNumber": "16",
       "text": "Rule 18: Do not redact Names and Addresses if Published Information found Do not redact Names and Addresses if Published Information found Lorem ipsum dolor sit amet Oxford University Press in voluptate velit esse cillum. Iakovos Geiger, Julian Ritter, Asya Lyon, Carina Madsen, Alexandra Häusler, Hanke Mendel, Ranya Eikenboom. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Min Kwok, Jun K., Tu Wong, Qiang Suen, Zhou Mah, Ning Liu, Lei W. Huang, Ru X. Wu"
     },
     {
-      "sectionNumber": 17,
+      "sectionNumber": "17",
       "text": "Rule 19/20: Redacted PII Personal Identification Information based on Dict Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No Naka-27 Aomachi, Nomi, Ishikawa 923-1101, Japan, JP Sude Halide Nurullah Özgür U. Reyhan B. Rahim C. J. Alfred Xinyi Y. Tao Clara Siegfried ← not added to Dict, should be not annotated"
     },
     {
-      "sectionNumber": 18,
+      "sectionNumber": "18",
       "text": "Rule 21/22: Redact Emails by RegEx Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No Duis aute irure dolor in library@outlook.com reprehenderit in voluptate gordonjcp@msn.com velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint dinther@comcast.net occaecat cupidatat non proident, sunt in kawasaki@me.com culpa qui officia deserunt mollit anim id est laborum."
     },
     {
-      "sectionNumber": 19,
+      "sectionNumber": "19",
       "text": "Description Text Contact Point"
     },
     {
-      "sectionNumber": 20,
+      "sectionNumber": "20",
       "text": "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum Contact Point dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Contact point: Central Research Industry Phone: +49 2113 2311 563 Fax: +49 2113 2311 560 Tel.: +81 764770164 Tel: +81 6653 44563 E-mail: Seriknowmobil@co.uk Email: maximiliamschmitt@arcor.de e-mail: maximiliamschmitt@t-online.de E-mail address: example@mail.com Contact: Maximiliam Schmitt Telephone number: +27414328992 Telephone No: +274 1432 8991 Fax number: +274 1432 8990 Telephone: +274 34223331 Phone No. +274 1432 8933 Contact: 493 1223 4592 European contact: European Central Institute Alternative contact: Emilia Lockhart Alternative contact: Cyberdyne Systems Tower Defense 121a Hong Kong, BT District"
     },
     {
-      "sectionNumber": 22,
+      "sectionNumber": "22",
       "text": "Rule 23/24: Redact contact information (contains \"Contact point:\") Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No “Contact-Information was found should be appears”"
     },
     {
-      "sectionNumber": 23,
+      "sectionNumber": "23",
       "text": "Description Text Applicant"
     },
     {
-      "sectionNumber": 24,
+      "sectionNumber": "24",
       "text": "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum Contact Point dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Contact point: Central Research Industry Phone: +49 2113 2311 563 Fax: +49 2113 2311 560 Tel.: +81 764770164 Tel: +81 6653 44563 E-mail: Seriknowmobil@co.uk Email: maximiliamschmitt@arcor.de e-mail: maximiliamschmitt@t-online.de E-mail address: example@mail.com Contact: Maximiliam Schmitt Telephone number: +27414328992 Telephone No: +274 1432 8991 Fax number: +274 1432 8990 Telephone: +274 34223331 Phone No. +274 1432 8933 Contact: 493 1223 4592 European contact: European Central Institute Alternative contact: Emilia Lockhart Alternative contact: Cyberdyne Systems Tower Defense 121a Hong Kong, BT District"
     },
     {
-      "sectionNumber": 26,
+      "sectionNumber": "26",
       "text": "Rule 25/26: Redact contact information (contains \"Applicant\" as Headline or Text) Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No “Applicant Information was found should be appears” Applicant Name: Soylent Corporation Contact point: Riddley Scott Address: 359-21 Huam-dong Yongsan-gu Seoul, South Korea Phone: +82 122 34188 Fax: +82 122 34180 E-mail: food-industry@korea.com Contact: This is a special case, everything between this and the next keyword should be redacted Tel.: +275 5678 1234 132 fsdfdfre frefref"
     },
     {
-      "sectionNumber": 27,
+      "sectionNumber": "27",
       "text": "Rule 27/28: Redact contact Information (contains Producer) Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No “Producer was found” should be appears Producer of the plant production Name: Umbrella Corporation Contact: Jill Valentine Address: 359-21 Huam-dong Yongsan-gu Seoul, South Korea Phone: +82 122 34188 Fax: +82 122 34180 E-mail: pharma-industry@korea.com"
     },
     {
-      "sectionNumber": 28,
+      "sectionNumber": "28",
       "text": "Rule 29/30/31/32: If Text contains \"AUTHORS:\" and \"COMPLETION DATES\" but not \"STUDY COMPLETION DATES\", then Redact between both Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No Study Report___ AUTHOR(S): Dr. Alan Grant COMPLETION DATE: 02 December 1997"
     },
     {
-      "sectionNumber": 29,
+      "sectionNumber": "29",
       "text": "Rule 29/30/31/32: (additional) negative Test for Study completion dates No Redaction should be appears here Study Report___ AUTHOR(S): Dr. Alan Grant STUDY COMPLETION DATE: 02 December 1997"
     },
     {
-      "sectionNumber": 30,
+      "sectionNumber": "30",
       "text": "Rule 33/34: If Text contains \"Performing Lab\" and \"Lab Project ID\", then Redact everything between Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No Study Report___ PERFORMING LABORATORY: Umbrella Corporation LABORATORY PROJECT ID: Number 20201/33991/ERZAT/21"
     },
     {
-      "sectionNumber": 31,
+      "sectionNumber": "31",
       "text": "Rule 35/36/37/38: ?? Tba"
     },
     {
-      "sectionNumber": 32,
+      "sectionNumber": "32",
       "text": "Rule 39: Purity Hint Add Purity as Hint when Percent-Numbers is there Test Item: Soda Purity: 45% ← should be Hint Purity: <45% ← should be Hint Purity: >45% ← should be Hint Purity: 101% ← should ne be Hint because >100 % is not possible Purity: =>45% ← should be not Hint because additional symbols Purity: =<45% ← should be not Hint because additional symbols Purity: aa 45% ← should be not Hint because additional symbols Purity: 45% aa ← should be not Hint because additional symbols Purity: aa45% ← should be not Hint because additional symbols Purity: 45%aa ← should be not Hint because additional symbols Product-Code: EAK-L443 purity: 99% ← not Hint because case sensitive purity: >99% ← not Hint because case sensitive purity: <99% ← not Hint because case sensitive Supplier: GreenForce"
     },
     {
-      "sectionNumber": 33,
+      "sectionNumber": "33",
       "text": "Rule 40: Ignore Dossier-Redaction if Confidentiality is not set Dont redact Dossier-Redaction if Confidentiality is not set in file attributes Excepteur sint occaecat cupidatat non proident, myDossierRedaction sunt in culpa qui officia deserunt mollit anim id est laborum."
     },
     {
-      "sectionNumber": 34,
+      "sectionNumber": "34",
       "text": "Rule 41/42: Redact Signatures Redact when Vertebrate Study is Yes Redact when Vertebrate Study is No __________________________ __________________________ Signed by: Dilara Sonnenschein Signed by: Tobias Müller"
     },
     {
-      "sectionNumber": 35,
+      "sectionNumber": "35.1.1.3",
       "text": "Rule 43: Redact Logo Redact Logo only if Vertebrate Study is Yes, else do nothing (skipped)"
     },
     {
-      "sectionNumber": 36,
+      "sectionNumber": "36",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 37,
+      "sectionNumber": "37",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 38,
+      "sectionNumber": "38",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 39,
+      "sectionNumber": "39",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 40,
+      "sectionNumber": "40",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 41,
+      "sectionNumber": "41",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 42,
+      "sectionNumber": "42",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 43,
+      "sectionNumber": "43",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 44,
+      "sectionNumber": "44",
       "text": "This is a Page-Header"
     },
     {
-      "sectionNumber": 45,
+      "sectionNumber": "45",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 46,
+      "sectionNumber": "46",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 47,
+      "sectionNumber": "47",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 48,
+      "sectionNumber": "48",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 49,
+      "sectionNumber": "49",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 50,
+      "sectionNumber": "50",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 51,
+      "sectionNumber": "51",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 52,
+      "sectionNumber": "52",
       "text": "This is a Page-Footer"
     },
     {
-      "sectionNumber": 53,
+      "sectionNumber": "53",
       "text": "This is a Page-Footer"
     }
   ]
-}
+}