diff --git a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/IndexMessageType.java b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/IndexMessageType.java index 35cf09b..542532f 100644 --- a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/IndexMessageType.java +++ b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/IndexMessageType.java @@ -1,5 +1,5 @@ package com.iqser.red.service.search.v1.model; public enum IndexMessageType { - INSERT, UPDATE; + INSERT, UPDATE, DROP; } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java index d567c42..8de6d49 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java @@ -4,6 +4,8 @@ import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; + import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -16,7 +18,7 @@ import org.springframework.context.annotation.Import; @Import({DefaultWebMvcConfiguration.class}) @EnableFeignClients(basePackageClasses = FileStatusClient.class) -@EnableConfigurationProperties(ElasticsearchSettings.class) +@EnableConfigurationProperties({ElasticsearchSettings.class, SearchServiceSettings.class}) @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class}) public class Application { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java index 49f89b7..ca179fd 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java @@ -6,13 +6,11 @@ import lombok.RequiredArgsConstructor; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; -import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.message.BasicHeader; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index a10aab3..d884af2 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -1,7 +1,21 @@ package com.iqser.red.service.search.v1.server.queue; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_DLQ; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_QUEUE; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_DQL; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_QUEUE; + +import java.util.List; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.search.v1.model.IndexMessage; import com.iqser.red.service.search.v1.model.IndexMessageType; import com.iqser.red.service.search.v1.server.client.DossierClient; @@ -11,14 +25,12 @@ import com.iqser.red.service.search.v1.server.model.Text; import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; import com.iqser.red.service.search.v1.server.service.DocumentIndexService; import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import com.iqser.red.service.search.v1.server.service.TextStorageService; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.rabbit.annotation.RabbitHandler; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.stereotype.Service; - -import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.*; @Slf4j @Service @@ -33,6 +45,9 @@ public class IndexingMessageReceiver { private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient; private final DocumentDeleteService documentDeleteService; private final DocumentUpdateService documentUpdateService; + private final IndexCreatorService indexCreatorService; + private final RabbitTemplate rabbitTemplate; + private final IndexDeleteService indexDeleteService; @RabbitHandler @@ -41,19 +56,33 @@ public class IndexingMessageReceiver { var indexRequest = objectMapper.readValue(in, IndexMessage.class); log.info("Processing indexing request: {}", indexRequest); - var fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); - var dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); - if (IndexMessageType.INSERT.equals(indexRequest.getMessageType())) { - fileStatusProcessingUpdateClient.indexing(indexRequest.getDossierId(), indexRequest.getFileId()); - Text text = textStorageService.getText(indexRequest.getDossierId(), indexRequest.getFileId()); - documentIndexService.indexDocument(indexRequest.getDossierTemplateId(), indexRequest.getDossierId(), indexRequest.getFileId(), fileStatus.getFilename(), text, fileStatus.getAssignee(), dossier.getStatus(), fileStatus.getWorkflowStatus(), fileStatus.getFileAttributes()); - fileStatusProcessingUpdateClient.indexingSuccessful(indexRequest.getDossierId(), indexRequest.getFileId()); - log.info("Successfully indexed {}", indexRequest); - } else if (IndexMessageType.UPDATE.equals(indexRequest.getMessageType())) { - documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), dossier.getStatus(), fileStatus.getWorkflowStatus().name(), fileStatus.getFileAttributes()); - log.info("Successfully updated {}", indexRequest); + switch (indexRequest.getMessageType()) { + case INSERT: + var fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); + var dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); + indexFile(dossier, fileStatus); + break; + + case UPDATE: + fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); + dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); + documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), dossier.getStatus(), fileStatus.getWorkflowStatus() + .name(), fileStatus.getFileAttributes()); + log.info("Successfully updated {}", indexRequest); + break; + + case DROP: + indexDeleteService.closeIndex(); + indexDeleteService.dropIndex(); + indexCreatorService.createIndex(); + addAllDocumentsToIndexQueue(); + break; + + default: + throw new IllegalArgumentException("MessageType '" + indexRequest.getMessageType() + "' does not exist"); } + } @@ -89,4 +118,44 @@ public class IndexingMessageReceiver { } + private void indexFile(Dossier dossier, FileModel file) { + + fileStatusProcessingUpdateClient.indexing(dossier.getId(), file.getId()); + Text text = textStorageService.getText(dossier.getId(), file.getId()); + documentIndexService.indexDocument(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), text, file.getAssignee(), dossier.getStatus(), file.getWorkflowStatus(), file.getFileAttributes()); + fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId()); + log.info("Successfully indexed dossier {} file {}", dossier.getId(), file.getId()); + } + + + private void addAllDocumentsToIndexQueue() { + + var allDossiers = dossierClient.getAllDossiers(true, true); + for (Dossier dossier : allDossiers) { + addFilesToIndexingQueue(dossier.getId(), fileStatusClient.getDossierStatus(dossier.getId())); + addFilesToIndexingQueue(dossier.getId(), fileStatusClient.getSoftDeletedDossierStatus(dossier.getId())); + } + log.info("Successfully added all files from all dossiers to index queue (including archived and deleted)"); + } + + + private void addFilesToIndexingQueue(String dossierId, List files) { + + for (FileModel file : files) { + try { + log.info("Will add dossier {} file {} to index queue", dossierId, file.getId()); + rabbitTemplate.convertAndSend(INDEXING_QUEUE, objectMapper.writeValueAsString(IndexMessage.builder() + .messageType(IndexMessageType.INSERT) + .dossierId(dossierId) + .fileId(file.getId()) + .build()), message -> { + message.getMessageProperties().setPriority(99); + return message; + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java index ed5e66a..4851d1f 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java @@ -1,20 +1,6 @@ package com.iqser.red.service.search.v1.server.service; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStats; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.model.*; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.common.xcontent.XContentType; -import org.springframework.stereotype.Service; +import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; import java.io.IOException; import java.time.OffsetDateTime; @@ -23,7 +9,27 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.common.xcontent.XContentType; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.model.IndexFileAttribute; +import com.iqser.red.service.search.v1.server.model.IndexSection; +import com.iqser.red.service.search.v1.server.model.SectionArea; +import com.iqser.red.service.search.v1.server.model.SectionText; +import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @@ -35,7 +41,8 @@ public class DocumentIndexService { private final ObjectMapper objectMapper; - public void indexDocument(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus, WorkflowStatus workflowStatus, Map fileAttributes) { + public void indexDocument(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus, + WorkflowStatus workflowStatus, Map fileAttributes) { IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id(fileId); indexRequest.setRefreshPolicy(settings.getRefreshPolicy()); @@ -59,7 +66,8 @@ public class DocumentIndexService { } - private IndexDocument convert(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus, WorkflowStatus workflowStatus, Map fileAttributes) { + private IndexDocument convert(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, DossierStatus dossierStatus, + WorkflowStatus workflowStatus, Map fileAttributes) { return IndexDocument.builder() .dossierTemplateId(dossierTemplateId) @@ -75,7 +83,8 @@ public class DocumentIndexService { .build(); } - private List convertFileAttributes(Map fileAttributes){ + + private List convertFileAttributes(Map fileAttributes) { List converted = new ArrayList<>(); fileAttributes.entrySet().forEach(entry -> converted.add(new IndexFileAttribute(entry.getKey(), entry.getValue()))); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java new file mode 100644 index 0000000..955c6cb --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.search.v1.server.service; + +import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.CloseIndexRequest; +import org.elasticsearch.core.TimeValue; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class IndexDeleteService { + + private final ElasticsearchClient client; + + + @SneakyThrows + public void closeIndex() { + + log.info("Will close index"); + CloseIndexRequest request = new CloseIndexRequest(INDEX_NAME); + request.setTimeout(TimeValue.timeValueMinutes(2)); + AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT); + + if (closeIndexResponse.isAcknowledged()) { + log.info("Index is closed"); + } else { + throw new ElasticsearchException("Error while closing index"); + } + + } + + + @SneakyThrows + public void dropIndex() { + + log.info("Will drop index"); + DeleteIndexRequest request = new DeleteIndexRequest(INDEX_NAME); + request.timeout(TimeValue.timeValueMinutes(2)); + request.timeout("2m"); + AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT); + + if (deleteIndexResponse.isAcknowledged()) { + log.info("Index is dropped"); + } else { + throw new ElasticsearchException("Error while dropping index"); + } + + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java new file mode 100644 index 0000000..5a0863b --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java @@ -0,0 +1,21 @@ +package com.iqser.red.service.search.v1.server.settings; + +import java.util.ArrayList; +import java.util.List; + +import org.elasticsearch.action.support.WriteRequest; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import lombok.Data; + +@Data +@Primary +@Configuration +@ConfigurationProperties("search-service") +public class SearchServiceSettings { + + private boolean dropAndRecreateIndex; + +} diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java index 254d08e..bd7ee36 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java @@ -12,6 +12,7 @@ import com.iqser.red.service.search.v1.server.model.Text; import lombok.SneakyThrows; import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -54,6 +55,12 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { @MockBean private DossierClient dossierClient; + @MockBean + private RabbitTemplate rabbitTemplate; + + @MockBean + private IndexDeleteService indexDeleteService; + private final long UPDATE_TIMER = 1500;