Marmelator/search-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

RED-3621: Added automated migration step to close, drop, recreate and reindex all files when index mapping has changed

Browse Source
This commit is contained in:
Philipp Schramm 2022-04-04 13:54:42 +02:00
parent 3a7eef94cf
commit 53b38aeab8
8 changed files with 269 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
search-service-v1/search-service-server-v1/pom.xml
View File

@ -12,7 +12,7 @@
<artifactId>search-service-server-v1</artifactId>
<properties>
<persistence-service.version>1.96.0</persistence-service.version>
<persistence-service.version>1.112.0</persistence-service.version>
</properties>
<dependencies>

10
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/IndexInformationClient.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.search.v1.server.client;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.persistence.service.v1.api.resources.IndexInformationResource;
@FeignClient(name = "IndexInformationResource", url = "${persistence-service.url}")
public interface IndexInformationClient extends IndexInformationResource {
}

1
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java
View File

@ -5,6 +5,7 @@ import com.iqser.red.service.search.v1.model.SearchResult;
import com.iqser.red.service.search.v1.resources.SearchResource;
import com.iqser.red.service.search.v1.server.service.SearchService;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;

40
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java Normal file
View File

@ -0,0 +1,40 @@
package com.iqser.red.service.search.v1.server.migration;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationContext;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.search.v1.model.IndexMessage;
import com.iqser.red.service.search.v1.model.IndexMessageType;
import com.iqser.red.service.search.v1.server.queue.IndexingMessageReceiver;
import com.iqser.red.service.search.v1.server.service.IndexInformationService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class MigrationStarterService {
private final ApplicationContext ctx;
private final IndexInformationService indexInformationService;
private final IndexingMessageReceiver indexingMessageReceiver;
private final ObjectMapper objectMapper;
@EventListener(ApplicationReadyEvent.class)
public void migrate() throws JsonProcessingException {
if (indexInformationService.hasIndexChanged()) {
log.info("Index has changed and will be closed, dropped, recreated and all files will be indexed");
String indexMessage = objectMapper.writeValueAsString(IndexMessage.builder().messageType(IndexMessageType.DROP).build());
indexingMessageReceiver.receiveIndexingRequest(indexMessage);
}
}
}

43
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java
View File

@ -1,5 +1,17 @@
package com.iqser.red.service.search.v1.server.queue;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_DLQ;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.DELETE_FROM_INDEX_QUEUE;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_DQL;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_QUEUE;
import java.util.List;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
@ -10,17 +22,16 @@ import com.iqser.red.service.search.v1.server.client.DossierClient;
import com.iqser.red.service.search.v1.server.client.FileStatusClient;
import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.search.v1.server.model.Text;
import com.iqser.red.service.search.v1.server.service.*;
import com.iqser.red.service.search.v1.server.service.DocumentDeleteService;
import com.iqser.red.service.search.v1.server.service.DocumentIndexService;
import com.iqser.red.service.search.v1.server.service.DocumentUpdateService;
import com.iqser.red.service.search.v1.server.service.IndexCreatorService;
import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
import com.iqser.red.service.search.v1.server.service.IndexInformationService;
import com.iqser.red.service.search.v1.server.service.TextStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import java.util.List;
import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.*;
@Slf4j
@Service
@ -38,6 +49,7 @@ public class IndexingMessageReceiver {
private final IndexCreatorService indexCreatorService;
private final RabbitTemplate rabbitTemplate;
private final IndexDeleteService indexDeleteService;
private final IndexInformationService indexInformationService;
@RabbitHandler
@ -57,9 +69,8 @@ public class IndexingMessageReceiver {
case UPDATE:
fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true);
documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(),
dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, fileStatus.getWorkflowStatus()
.name(), fileStatus.getFileAttributes());
documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, fileStatus.getWorkflowStatus()
.name(), fileStatus.getFileAttributes());
log.info("Successfully updated {}", indexRequest);
break;
@ -68,6 +79,11 @@ public class IndexingMessageReceiver {
indexDeleteService.dropIndex();
indexCreatorService.createIndex();
addAllDocumentsToIndexQueue();
try {
indexInformationService.updateIndexInformation();
} catch (Exception e) {
log.error("Could not update index information", e);
}
break;
default:
@ -113,8 +129,7 @@ public class IndexingMessageReceiver {
fileStatusProcessingUpdateClient.indexing(dossier.getId(), file.getId());
Text text = textStorageService.getText(dossier.getId(), file.getId());
documentIndexService.indexDocument(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), text, file.getAssignee(),
dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, file.getWorkflowStatus(), file.getFileAttributes());
documentIndexService.indexDocument(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), text, file.getAssignee(), dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, file.getWorkflowStatus(), file.getFileAttributes());
fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId());
log.info("Successfully indexed dossier {} file {}", dossier.getId(), file.getId());
}

77
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java Normal file
View File

@ -0,0 +1,77 @@
package com.iqser.red.service.search.v1.server.service;
import java.io.BufferedInputStream;
import java.security.MessageDigest;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import org.apache.commons.codec.binary.StringUtils;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.index.IndexInformation;
import com.iqser.red.service.search.v1.server.client.IndexInformationClient;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class IndexInformationService {
private static final String PATH_TO_CONFIG = "index/mapping.json";
private final IndexInformationClient indexInformationClient;
public boolean hasIndexChanged() {
try {
IndexInformation indexInformationFromDatabase = indexInformationClient.getIndexInformation();
if (indexInformationFromDatabase == null) {
return true;
}
String fileHash = generateIndexConfigurationHash();
log.info("Hash from database {} (updated {}) and hash from file {}", indexInformationFromDatabase.getIndexConfigurationHash(), indexInformationFromDatabase.getUpdateDate(), fileHash);
if (StringUtils.equals(indexInformationFromDatabase.getIndexConfigurationHash(), fileHash)) {
return false;
}
} catch (Exception e) {
log.error("Exception while comparing index hashes", e);
}
return true;
}
public void updateIndexInformation() {
IndexInformation indexInformation = IndexInformation.builder()
.indexConfigurationHash(generateIndexConfigurationHash())
.updateDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))
.build();
indexInformationClient.updateIndexInformation(indexInformation);
}
@SneakyThrows
public String generateIndexConfigurationHash() {
byte[] buffer = new byte[8192];
int count;
MessageDigest digest = MessageDigest.getInstance("SHA-256");
BufferedInputStream bis = new BufferedInputStream(new ClassPathResource(PATH_TO_CONFIG).getInputStream());
while ((count = bis.read(buffer)) > 0) {
digest.update(buffer, 0, count);
}
bis.close();
return Arrays.toString(digest.digest());
}
}

85
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/IndexTest.java Normal file
View File

@ -0,0 +1,85 @@
package com.iqser.red.service.search.v1.server.service;
import static org.mockito.Mockito.when;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit4.SpringRunner;
import com.iqser.red.service.persistence.service.v1.api.model.index.IndexInformation;
import com.iqser.red.service.search.v1.server.client.FileStatusClient;
import com.iqser.red.service.search.v1.server.client.IndexInformationClient;
import com.iqser.red.service.search.v1.server.queue.IndexingMessageReceiver;
import lombok.SneakyThrows;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS})
public class IndexTest extends AbstractElasticsearchIntegrationTest {
@Autowired
private IndexInformationService indexInformationService;
@MockBean
private IndexInformationClient indexInformationClient;
@MockBean
private FileStatusClient fileStatusClient;
@MockBean
private IndexingMessageReceiver indexingMessageReceiver;
@Test
@SneakyThrows
public void testGenerateHash() {
// Act
String hash = indexInformationService.generateIndexConfigurationHash();
// Assert
System.out.println(hash);
Assert.assertNotNull(hash);
}
@Test
@SneakyThrows
public void testHashChanged() {
// Arrange
IndexInformation indexInformation = IndexInformation.builder().indexConfigurationHash("Some Hash").build();
when(indexInformationClient.getIndexInformation()).thenReturn(indexInformation);
// Act and Assert
Assert.assertTrue(indexInformationService.hasIndexChanged());
}
@Test
@SneakyThrows
public void testHashChangedNot() {
// Arrange
String hash = indexInformationService.generateIndexConfigurationHash();
IndexInformation indexInformation = IndexInformation.builder().indexConfigurationHash(hash).build();
when(indexInformationClient.getIndexInformation()).thenReturn(indexInformation);
// Act and Assert
Assert.assertFalse(indexInformationService.hasIndexChanged());
}
@Test
@SneakyThrows
public void testHashDoesNotExist() {
// Arrange
when(indexInformationClient.getIndexInformation()).thenReturn(null);
// Act and Assert
Assert.assertTrue(indexInformationService.hasIndexChanged());
}
}

50
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java
View File

@ -1,16 +1,12 @@
package com.iqser.red.service.search.v1.server.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.search.v1.model.MatchedDocument;
import com.iqser.red.service.search.v1.model.SearchResult;
import com.iqser.red.service.search.v1.server.client.DossierClient;
import com.iqser.red.service.search.v1.server.client.FileStatusClient;
import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.search.v1.server.model.Text;
import lombok.SneakyThrows;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
@ -21,12 +17,17 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit4.SpringRunner;
import org.testcontainers.shaded.org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.search.v1.model.MatchedDocument;
import com.iqser.red.service.search.v1.model.SearchResult;
import com.iqser.red.service.search.v1.server.client.DossierClient;
import com.iqser.red.service.search.v1.server.client.FileStatusClient;
import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.search.v1.server.client.IndexInformationClient;
import com.iqser.red.service.search.v1.server.model.Text;
import static org.assertj.core.api.Assertions.assertThat;
import lombok.SneakyThrows;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS})
@ -62,6 +63,12 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest {
@MockBean
private IndexDeleteService indexDeleteService;
@MockBean
private IndexInformationClient indexInformationClient;
@MockBean
private IndexInformationService indexInformationService;
private final long UPDATE_TIMER = 1500;
@ -81,6 +88,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest {
}
@Test
@SneakyThrows
public void testSearchWithAllFilter() {
@ -137,6 +145,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest {
assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2");
}
/*
* Index two documents, update one and filter by assignee
*/
@ -584,16 +593,9 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest {
// Assert
assertThat(result.getMatchedDocuments().size()).isEqualTo(1);
assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0);
assertThat(result.getMatchedDocuments()
.get(0)
.getMatchedTerms()
.contains(searchString)).isTrue();
assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue();
assertThat(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").size()).isGreaterThan(0);
assertThat(StringUtils.contains(result.getMatchedDocuments()
.get(0)
.getHighlights()
.get("sections.text")
.toArray()[0].toString(), searchString)).isTrue();
assertThat(StringUtils.contains(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").toArray()[0].toString(), searchString)).isTrue();
}
}