diff --git a/search-service-v1/search-service-server-v1/pom.xml b/search-service-v1/search-service-server-v1/pom.xml index 54a529e..a7ccc6f 100644 --- a/search-service-v1/search-service-server-v1/pom.xml +++ b/search-service-v1/search-service-server-v1/pom.xml @@ -12,7 +12,7 @@ search-service-server-v1 - 1.96.0 + 1.112.0 diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/IndexInformationClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/IndexInformationClient.java new file mode 100644 index 0000000..54dbeb4 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/IndexInformationClient.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.search.v1.server.client; + +import org.springframework.cloud.openfeign.FeignClient; + +import com.iqser.red.service.persistence.service.v1.api.resources.IndexInformationResource; + +@FeignClient(name = "IndexInformationResource", url = "${persistence-service.url}") +public interface IndexInformationClient extends IndexInformationResource { + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java index a27f10a..ffd27bb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java @@ -5,6 +5,7 @@ import com.iqser.red.service.search.v1.model.SearchResult; import com.iqser.red.service.search.v1.resources.SearchResource; import com.iqser.red.service.search.v1.server.service.SearchService; import lombok.RequiredArgsConstructor; + import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java new file mode 100644 index 0000000..2bee23b --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java @@ -0,0 +1,40 @@ +package com.iqser.red.service.search.v1.server.migration; + +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationContext; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.search.v1.model.IndexMessage; +import com.iqser.red.service.search.v1.model.IndexMessageType; +import com.iqser.red.service.search.v1.server.queue.IndexingMessageReceiver; +import com.iqser.red.service.search.v1.server.service.IndexInformationService; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class MigrationStarterService { + + private final ApplicationContext ctx; + private final IndexInformationService indexInformationService; + private final IndexingMessageReceiver indexingMessageReceiver; + private final ObjectMapper objectMapper; + + + @EventListener(ApplicationReadyEvent.class) + public void migrate() throws JsonProcessingException { + + if (indexInformationService.hasIndexChanged()) { + log.info("Index has changed and will be closed, dropped, recreated and all files will be indexed"); + String indexMessage = objectMapper.writeValueAsString(IndexMessage.builder().messageType(IndexMessageType.DROP).build()); + indexingMessageReceiver.receiveIndexingRequest(indexMessage); + } + + } + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index 16f8479..5f48919 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -1,5 +1,17 @@ package com.iqser.red.service.search.v1.server.queue; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_DLQ; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_QUEUE; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_DQL; +import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_QUEUE; + +import java.util.List; + +import org.springframework.amqp.rabbit.annotation.RabbitHandler; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; @@ -10,17 +22,16 @@ import com.iqser.red.service.search.v1.server.client.DossierClient; import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.search.v1.server.model.Text; -import com.iqser.red.service.search.v1.server.service.*; +import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; +import com.iqser.red.service.search.v1.server.service.DocumentIndexService; +import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; +import com.iqser.red.service.search.v1.server.service.IndexInformationService; +import com.iqser.red.service.search.v1.server.service.TextStorageService; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.rabbit.annotation.RabbitHandler; -import org.springframework.amqp.rabbit.annotation.RabbitListener; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - -import java.util.List; - -import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.*; @Slf4j @Service @@ -38,6 +49,7 @@ public class IndexingMessageReceiver { private final IndexCreatorService indexCreatorService; private final RabbitTemplate rabbitTemplate; private final IndexDeleteService indexDeleteService; + private final IndexInformationService indexInformationService; @RabbitHandler @@ -57,9 +69,8 @@ public class IndexingMessageReceiver { case UPDATE: fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); - documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), - dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, fileStatus.getWorkflowStatus() - .name(), fileStatus.getFileAttributes()); + documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, fileStatus.getWorkflowStatus() + .name(), fileStatus.getFileAttributes()); log.info("Successfully updated {}", indexRequest); break; @@ -68,6 +79,11 @@ public class IndexingMessageReceiver { indexDeleteService.dropIndex(); indexCreatorService.createIndex(); addAllDocumentsToIndexQueue(); + try { + indexInformationService.updateIndexInformation(); + } catch (Exception e) { + log.error("Could not update index information", e); + } break; default: @@ -113,8 +129,7 @@ public class IndexingMessageReceiver { fileStatusProcessingUpdateClient.indexing(dossier.getId(), file.getId()); Text text = textStorageService.getText(dossier.getId(), file.getId()); - documentIndexService.indexDocument(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), text, file.getAssignee(), - dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, file.getWorkflowStatus(), file.getFileAttributes()); + documentIndexService.indexDocument(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), text, file.getAssignee(), dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, file.getWorkflowStatus(), file.getFileAttributes()); fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId()); log.info("Successfully indexed dossier {} file {}", dossier.getId(), file.getId()); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java new file mode 100644 index 0000000..18198fd --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java @@ -0,0 +1,77 @@ +package com.iqser.red.service.search.v1.server.service; + +import java.io.BufferedInputStream; +import java.security.MessageDigest; +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; + +import org.apache.commons.codec.binary.StringUtils; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.model.index.IndexInformation; +import com.iqser.red.service.search.v1.server.client.IndexInformationClient; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class IndexInformationService { + + private static final String PATH_TO_CONFIG = "index/mapping.json"; + + private final IndexInformationClient indexInformationClient; + + + public boolean hasIndexChanged() { + + try { + IndexInformation indexInformationFromDatabase = indexInformationClient.getIndexInformation(); + if (indexInformationFromDatabase == null) { + return true; + } + String fileHash = generateIndexConfigurationHash(); + log.info("Hash from database {} (updated {}) and hash from file {}", indexInformationFromDatabase.getIndexConfigurationHash(), indexInformationFromDatabase.getUpdateDate(), fileHash); + + if (StringUtils.equals(indexInformationFromDatabase.getIndexConfigurationHash(), fileHash)) { + return false; + } + + } catch (Exception e) { + log.error("Exception while comparing index hashes", e); + } + return true; + } + + + public void updateIndexInformation() { + + IndexInformation indexInformation = IndexInformation.builder() + .indexConfigurationHash(generateIndexConfigurationHash()) + .updateDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)) + .build(); + + indexInformationClient.updateIndexInformation(indexInformation); + } + + + @SneakyThrows + public String generateIndexConfigurationHash() { + + byte[] buffer = new byte[8192]; + int count; + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + BufferedInputStream bis = new BufferedInputStream(new ClassPathResource(PATH_TO_CONFIG).getInputStream()); + while ((count = bis.read(buffer)) > 0) { + digest.update(buffer, 0, count); + } + bis.close(); + + return Arrays.toString(digest.digest()); + } + +} diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/IndexTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/IndexTest.java new file mode 100644 index 0000000..1b2e6fa --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/IndexTest.java @@ -0,0 +1,85 @@ +package com.iqser.red.service.search.v1.server.service; + +import static org.mockito.Mockito.when; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit4.SpringRunner; + +import com.iqser.red.service.persistence.service.v1.api.model.index.IndexInformation; +import com.iqser.red.service.search.v1.server.client.FileStatusClient; +import com.iqser.red.service.search.v1.server.client.IndexInformationClient; +import com.iqser.red.service.search.v1.server.queue.IndexingMessageReceiver; + +import lombok.SneakyThrows; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS}) +public class IndexTest extends AbstractElasticsearchIntegrationTest { + + @Autowired + private IndexInformationService indexInformationService; + + @MockBean + private IndexInformationClient indexInformationClient; + + @MockBean + private FileStatusClient fileStatusClient; + + @MockBean + private IndexingMessageReceiver indexingMessageReceiver; + + + @Test + @SneakyThrows + public void testGenerateHash() { + // Act + String hash = indexInformationService.generateIndexConfigurationHash(); + + // Assert + System.out.println(hash); + Assert.assertNotNull(hash); + + } + + + @Test + @SneakyThrows + public void testHashChanged() { + // Arrange + IndexInformation indexInformation = IndexInformation.builder().indexConfigurationHash("Some Hash").build(); + when(indexInformationClient.getIndexInformation()).thenReturn(indexInformation); + + // Act and Assert + Assert.assertTrue(indexInformationService.hasIndexChanged()); + } + + + @Test + @SneakyThrows + public void testHashChangedNot() { + // Arrange + String hash = indexInformationService.generateIndexConfigurationHash(); + IndexInformation indexInformation = IndexInformation.builder().indexConfigurationHash(hash).build(); + when(indexInformationClient.getIndexInformation()).thenReturn(indexInformation); + + // Act and Assert + Assert.assertFalse(indexInformationService.hasIndexChanged()); + } + + + @Test + @SneakyThrows + public void testHashDoesNotExist() { + // Arrange + when(indexInformationClient.getIndexInformation()).thenReturn(null); + + // Act and Assert + Assert.assertTrue(indexInformationService.hasIndexChanged()); + } + +} diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java index b4c24eb..e5606ad 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java @@ -1,16 +1,12 @@ package com.iqser.red.service.search.v1.server.service; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.search.v1.model.MatchedDocument; -import com.iqser.red.service.search.v1.model.SearchResult; -import com.iqser.red.service.search.v1.server.client.DossierClient; -import com.iqser.red.service.search.v1.server.client.FileStatusClient; -import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; -import com.iqser.red.service.search.v1.server.model.Text; -import lombok.SneakyThrows; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.amqp.rabbit.core.RabbitTemplate; @@ -21,12 +17,17 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit4.SpringRunner; import org.testcontainers.shaded.org.apache.commons.lang.StringUtils; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.search.v1.model.MatchedDocument; +import com.iqser.red.service.search.v1.model.SearchResult; +import com.iqser.red.service.search.v1.server.client.DossierClient; +import com.iqser.red.service.search.v1.server.client.FileStatusClient; +import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; +import com.iqser.red.service.search.v1.server.client.IndexInformationClient; +import com.iqser.red.service.search.v1.server.model.Text; -import static org.assertj.core.api.Assertions.assertThat; +import lombok.SneakyThrows; @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS}) @@ -62,6 +63,12 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { @MockBean private IndexDeleteService indexDeleteService; + @MockBean + private IndexInformationClient indexInformationClient; + + @MockBean + private IndexInformationService indexInformationService; + private final long UPDATE_TIMER = 1500; @@ -81,6 +88,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { } + @Test @SneakyThrows public void testSearchWithAllFilter() { @@ -137,6 +145,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); } + /* * Index two documents, update one and filter by assignee */ @@ -584,16 +593,9 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { // Assert assertThat(result.getMatchedDocuments().size()).isEqualTo(1); assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); - assertThat(result.getMatchedDocuments() - .get(0) - .getMatchedTerms() - .contains(searchString)).isTrue(); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); assertThat(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").size()).isGreaterThan(0); - assertThat(StringUtils.contains(result.getMatchedDocuments() - .get(0) - .getHighlights() - .get("sections.text") - .toArray()[0].toString(), searchString)).isTrue(); + assertThat(StringUtils.contains(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").toArray()[0].toString(), searchString)).isTrue(); } }