Pull request #41: RED-3334

Merge in RED/search-service from RED-3334 to master * commit '763d505b021825389c7535dcff9031594190ce7d': RED-3334 Implemented deleting and rebuilding index
2022-03-14 10:20:39 +01:00 · 2022-03-14 10:20:39 +01:00 · f83cd10be5
commit f83cd10be5
parent 692f634d67 763d505b02
8 changed files with 206 additions and 39 deletions
--- a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/IndexMessageType.java
+++ b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/IndexMessageType.java
@ -1,5 +1,5 @@
 package com.iqser.red.service.search.v1.model;

 public enum IndexMessageType {
-    INSERT, UPDATE;
+    INSERT, UPDATE, DROP;
 }
--- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java
+++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java
@ -4,6 +4,8 @@ import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.search.v1.server.client.ElasticsearchClient;
 import com.iqser.red.service.search.v1.server.client.FileStatusClient;
 import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
+import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings;
+
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
@ -16,7 +18,7 @@ import org.springframework.context.annotation.Import;

@Import({DefaultWebMvcConfiguration.class})
@EnableFeignClients(basePackageClasses = FileStatusClient.class)
-@EnableConfigurationProperties(ElasticsearchSettings.class)
+@EnableConfigurationProperties({ElasticsearchSettings.class, SearchServiceSettings.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {

--- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java
+++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java
@ -6,13 +6,11 @@ import lombok.RequiredArgsConstructor;
 import lombok.experimental.Delegate;
 import lombok.extern.slf4j.Slf4j;

-import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.RestHighLevelClient;
--- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java
+++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java
@ -1,7 +1,21 @@
 package com.iqser.red.service.search.v1.server.queue;

+import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_DLQ;
+import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_QUEUE;
+import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_DQL;
+import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_QUEUE;
+
+import java.util.List;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
 import com.iqser.red.service.search.v1.model.IndexMessage;
 import com.iqser.red.service.search.v1.model.IndexMessageType;
 import com.iqser.red.service.search.v1.server.client.DossierClient;
@ -11,14 +25,12 @@ import com.iqser.red.service.search.v1.server.model.Text;
 import com.iqser.red.service.search.v1.server.service.DocumentDeleteService;
 import com.iqser.red.service.search.v1.server.service.DocumentIndexService;
 import com.iqser.red.service.search.v1.server.service.DocumentUpdateService;
+import com.iqser.red.service.search.v1.server.service.IndexCreatorService;
+import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
 import com.iqser.red.service.search.v1.server.service.TextStorageService;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.stereotype.Service;
-
-import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.*;

@Slf4j
@Service
@ -33,6 +45,9 @@ public class IndexingMessageReceiver {
    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
    private final DocumentDeleteService documentDeleteService;
    private final DocumentUpdateService documentUpdateService;
+    private final IndexCreatorService indexCreatorService;
+    private final RabbitTemplate rabbitTemplate;
+    private final IndexDeleteService indexDeleteService;


    @RabbitHandler
@ -41,19 +56,33 @@ public class IndexingMessageReceiver {

        var indexRequest = objectMapper.readValue(in, IndexMessage.class);
        log.info("Processing indexing request: {}", indexRequest);
-        var fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
-        var dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);

-        if (IndexMessageType.INSERT.equals(indexRequest.getMessageType())) {
-            fileStatusProcessingUpdateClient.indexing(indexRequest.getDossierId(), indexRequest.getFileId());
-            Text text = textStorageService.getText(indexRequest.getDossierId(), indexRequest.getFileId());
-            documentIndexService.indexDocument(indexRequest.getDossierTemplateId(), indexRequest.getDossierId(), indexRequest.getFileId(), fileStatus.getFilename(), text, fileStatus.getAssignee(), dossier.getStatus(), fileStatus.getWorkflowStatus(), fileStatus.getFileAttributes());
-            fileStatusProcessingUpdateClient.indexingSuccessful(indexRequest.getDossierId(), indexRequest.getFileId());
-            log.info("Successfully indexed {}", indexRequest);
-        } else if (IndexMessageType.UPDATE.equals(indexRequest.getMessageType())) {
-            documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), dossier.getStatus(), fileStatus.getWorkflowStatus().name(), fileStatus.getFileAttributes());
-            log.info("Successfully updated {}", indexRequest);
+        switch (indexRequest.getMessageType()) {
+            case INSERT:
+                var fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
+                var dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);
+                indexFile(dossier, fileStatus);
+                break;
+
+            case UPDATE:
+                fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
+                dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);
+                documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), dossier.getStatus(), fileStatus.getWorkflowStatus()
+                        .name(), fileStatus.getFileAttributes());
+                log.info("Successfully updated {}", indexRequest);
+                break;
+
+            case DROP:
+                indexDeleteService.closeIndex();
+                indexDeleteService.dropIndex();
+                indexCreatorService.createIndex();
+                addAllDocumentsToIndexQueue();
+                break;
+
+            default:
+                throw new IllegalArgumentException("MessageType '" + indexRequest.getMessageType() + "' does not exist");
        }
+
    }


@ -89,4 +118,44 @@ public class IndexingMessageReceiver {
    }


+    private void indexFile(Dossier dossier, FileModel file) {
+
+        fileStatusProcessingUpdateClient.indexing(dossier.getId(), file.getId());
+        Text text = textStorageService.getText(dossier.getId(), file.getId());
+        documentIndexService.indexDocument(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), text, file.getAssignee(), dossier.getStatus(), file.getWorkflowStatus(), file.getFileAttributes());
+        fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId());
+        log.info("Successfully indexed dossier {} file {}", dossier.getId(), file.getId());
+    }
+
+
+    private void addAllDocumentsToIndexQueue() {
+
+        var allDossiers = dossierClient.getAllDossiers(true, true);
+        for (Dossier dossier : allDossiers) {
+            addFilesToIndexingQueue(dossier.getId(), fileStatusClient.getDossierStatus(dossier.getId()));
+            addFilesToIndexingQueue(dossier.getId(), fileStatusClient.getSoftDeletedDossierStatus(dossier.getId()));
+        }
+        log.info("Successfully added all files from all dossiers to index queue (including archived and deleted)");
+    }
+
+
+    private void addFilesToIndexingQueue(String dossierId, List<FileModel> files) {
+
+        for (FileModel file : files) {
+            try {
+                log.info("Will add dossier {} file {} to index queue", dossierId, file.getId());
+                rabbitTemplate.convertAndSend(INDEXING_QUEUE, objectMapper.writeValueAsString(IndexMessage.builder()
+                        .messageType(IndexMessageType.INSERT)
+                        .dossierId(dossierId)
+                        .fileId(file.getId())
+                        .build()), message -> {
+                    message.getMessageProperties().setPriority(99);
+                    return message;
+                });
+            } catch (JsonProcessingException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
 }
--- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java
+++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java
@ -1,20 +1,6 @@
 package com.iqser.red.service.search.v1.server.service;

-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStats;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
-import com.iqser.red.service.search.v1.server.client.ElasticsearchClient;
-import com.iqser.red.service.search.v1.server.exception.IndexException;
-import com.iqser.red.service.search.v1.server.model.*;
-import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.springframework.stereotype.Service;
+import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME;

 import java.io.IOException;
 import java.time.OffsetDateTime;
@ -23,7 +9,27 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;

-import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
+import com.iqser.red.service.search.v1.server.client.ElasticsearchClient;
+import com.iqser.red.service.search.v1.server.exception.IndexException;
+import com.iqser.red.service.search.v1.server.model.IndexDocument;
+import com.iqser.red.service.search.v1.server.model.IndexFileAttribute;
+import com.iqser.red.service.search.v1.server.model.IndexSection;
+import com.iqser.red.service.search.v1.server.model.SectionArea;
+import com.iqser.red.service.search.v1.server.model.SectionText;
+import com.iqser.red.service.search.v1.server.model.Text;
+import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
@ -35,7 +41,8 @@ public class DocumentIndexService {
    private final ObjectMapper objectMapper;


-    public void indexDocument(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus, WorkflowStatus workflowStatus, Map<String, String> fileAttributes) {
+    public void indexDocument(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus,
+                              WorkflowStatus workflowStatus, Map<String, String> fileAttributes) {

        IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id(fileId);
        indexRequest.setRefreshPolicy(settings.getRefreshPolicy());
@ -59,7 +66,8 @@ public class DocumentIndexService {
    }


-    private IndexDocument convert(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus, WorkflowStatus workflowStatus, Map<String, String> fileAttributes) {
+    private IndexDocument convert(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus,
+                                  WorkflowStatus workflowStatus, Map<String, String> fileAttributes) {

        return IndexDocument.builder()
                .dossierTemplateId(dossierTemplateId)
@ -75,7 +83,8 @@ public class DocumentIndexService {
                .build();
    }

-    private List<IndexFileAttribute> convertFileAttributes(Map<String, String> fileAttributes){
+
+    private List<IndexFileAttribute> convertFileAttributes(Map<String, String> fileAttributes) {

        List<IndexFileAttribute> converted = new ArrayList<>();
        fileAttributes.entrySet().forEach(entry -> converted.add(new IndexFileAttribute(entry.getKey(), entry.getValue())));
--- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java
+++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java
@ -0,0 +1,61 @@
+package com.iqser.red.service.search.v1.server.service;
+
+import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.indices.CloseIndexRequest;
+import org.elasticsearch.core.TimeValue;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.search.v1.server.client.ElasticsearchClient;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class IndexDeleteService {
+
+    private final ElasticsearchClient client;
+
+
+    @SneakyThrows
+    public void closeIndex() {
+
+        log.info("Will close index");
+        CloseIndexRequest request = new CloseIndexRequest(INDEX_NAME);
+        request.setTimeout(TimeValue.timeValueMinutes(2));
+        AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT);
+
+        if (closeIndexResponse.isAcknowledged()) {
+            log.info("Index is closed");
+        } else {
+            throw new ElasticsearchException("Error while closing index");
+        }
+
+    }
+
+
+    @SneakyThrows
+    public void dropIndex() {
+
+        log.info("Will drop index");
+        DeleteIndexRequest request = new DeleteIndexRequest(INDEX_NAME);
+        request.timeout(TimeValue.timeValueMinutes(2));
+        request.timeout("2m");
+        AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT);
+
+        if (deleteIndexResponse.isAcknowledged()) {
+            log.info("Index is dropped");
+        } else {
+            throw new ElasticsearchException("Error while dropping index");
+        }
+
+    }
+
+}
--- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java
+++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java
@ -0,0 +1,21 @@
+package com.iqser.red.service.search.v1.server.settings;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.elasticsearch.action.support.WriteRequest;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+
+import lombok.Data;
+
+@Data
+@Primary
+@Configuration
+@ConfigurationProperties("search-service")
+public class SearchServiceSettings {
+
+    private boolean dropAndRecreateIndex;
+
+}
--- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java
+++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java
@ -12,6 +12,7 @@ import com.iqser.red.service.search.v1.server.model.Text;
 import lombok.SneakyThrows;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
@ -54,6 +55,12 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest {
    @MockBean
    private DossierClient dossierClient;

+    @MockBean
+    private RabbitTemplate rabbitTemplate;
+
+    @MockBean
+    private IndexDeleteService indexDeleteService;
+
    private final long UPDATE_TIMER = 1500;