From 41fbdaca7187587f7e25921a78a258f2e63ebef6 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Thu, 2 Mar 2023 08:53:44 +0100 Subject: [PATCH 1/6] RED-5669: Adapted service to new elasticseach api --- .../search/v1/model/MatchedDocument.java | 2 +- .../search-service-server-v1/pom.xml | 23 +- .../service/search/v1/server/Application.java | 31 +- .../server/controller/SearchController.java | 2 +- .../v1/server/exception/IndexException.java | 1 + .../search/v1/server/model/IndexDocument.java | 4 +- .../server/queue/IndexingMessageReceiver.java | 24 +- .../server/service/DocumentIndexService.java | 123 ------- .../server/service/DocumentUpdateService.java | 64 ---- .../server/service/IndexCreatorService.java | 75 ---- .../v1/server/service/IndexDeleteService.java | 61 ---- .../IndexDocumentConverterService.java | 84 +++++ .../v1/server/service/SearchService.java | 313 ---------------- .../DocumentDeleteService.java | 17 +- .../elasticsearch/DocumentIndexService.java | 44 +++ .../elasticsearch/DocumentUpdateService.java | 37 ++ .../elasticsearch/EsClient.java} | 29 +- .../elasticsearch/IndexCreatorService.java | 89 +++++ .../elasticsearch/IndexDeleteService.java | 46 +++ .../service/elasticsearch/SearchService.java | 333 ++++++++++++++++++ .../settings/ElasticsearchSettings.java | 8 +- .../v1/server/utils/ResourceLoader.java | 6 +- .../AbstractElasticsearchIntegrationTest.java | 7 +- .../search/v1/server/service/SearchTest.java | 149 +++++--- 24 files changed, 825 insertions(+), 747 deletions(-) delete mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java delete mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java delete mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java delete mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDocumentConverterService.java delete mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/{ => elasticsearch}/DocumentDeleteService.java (50%) create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/{client/ElasticsearchClient.java => service/elasticsearch/EsClient.java} (72%) create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java diff --git a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/MatchedDocument.java b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/MatchedDocument.java index cebed22..cd54703 100644 --- a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/MatchedDocument.java +++ b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/MatchedDocument.java @@ -32,7 +32,7 @@ public class MatchedDocument { private String fileName; @Builder.Default - private Map> highlights = new HashMap<>(); + private Map> highlights = new HashMap<>(); @Builder.Default private Set matchedTerms = new HashSet<>(); diff --git a/search-service-v1/search-service-server-v1/pom.xml b/search-service-v1/search-service-server-v1/pom.xml index d7f34a3..dfbd9ed 100644 --- a/search-service-v1/search-service-server-v1/pom.xml +++ b/search-service-v1/search-service-server-v1/pom.xml @@ -51,12 +51,29 @@ jackson-commons + - org.opensearch.client - opensearch-rest-high-level-client - 1.2.4 + co.elastic.clients + elasticsearch-java + 8.6.2 + + + + + + + jakarta.json + jakarta.json-api + 2.0.1 + + + + + + + org.springframework.cloud diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java index 80e76dd..611eb5c 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java @@ -1,5 +1,9 @@ package com.iqser.red.service.search.v1.server; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; + +import java.util.TimeZone; + import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -9,10 +13,15 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; import com.iqser.red.service.search.v1.server.client.FileStatusClient; +import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; @@ -32,10 +41,24 @@ public class Application { @Bean - @ConditionalOnMissingBean - public ElasticsearchClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { + @Primary + public ObjectMapper objectMapper() { - return new ElasticsearchClient(elasticsearchSettings); + return new ObjectMapper().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .enable(SerializationFeature.WRITE_DATES_WITH_ZONE_ID) + .disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE) + .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + .registerModule(new JavaTimeModule()) + .setTimeZone(TimeZone.getDefault()) + .findAndRegisterModules(); + } + + + @Bean + @ConditionalOnMissingBean + public EsClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { + + return new EsClient(elasticsearchSettings); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java index c5949b7..1d4a82e 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java @@ -6,7 +6,7 @@ import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.search.v1.model.SearchRequest; import com.iqser.red.service.search.v1.model.SearchResult; import com.iqser.red.service.search.v1.resources.SearchResource; -import com.iqser.red.service.search.v1.server.service.SearchService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService; import lombok.RequiredArgsConstructor; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java index 348419e..5b9a5f8 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java @@ -5,6 +5,7 @@ public class IndexException extends RuntimeException { public static final String INDEX_EXISTS_ERROR = "Unable to check, if index exists"; public static final String CONTENT_TO_JSON_ERROR = "Could not convert document with id '%s' to JSON!"; public static final String DOCUMENT_INDEX_ERROR = "Error during indexing document with id '%s'"; + public static final String DOCUMENT_UPDATE_ERROR = "Error during updating document with id '%s'"; public static final String DOCUMENT_DELETE_ERROR = "Error during deleting document with id '%s'"; public static final String FAILED_TO_SEARCH = "Error during search"; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java index eadeabb..3783778 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java @@ -1,8 +1,8 @@ package com.iqser.red.service.search.v1.server.model; import java.io.Serializable; -import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Date; import java.util.List; import lombok.AccessLevel; @@ -23,7 +23,7 @@ public class IndexDocument implements Serializable { private String fileId; private String filename; - private OffsetDateTime date; + private Date date; private String assignee; private boolean dossierDeleted; private boolean dossierArchived; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index 01a9ffe..33336e5 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -24,15 +24,15 @@ import com.iqser.red.service.search.v1.server.client.DossierClient; import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.search.v1.server.model.Text; -import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; -import com.iqser.red.service.search.v1.server.service.DocumentIndexService; -import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; -import com.iqser.red.service.search.v1.server.service.IndexCreatorService; -import com.iqser.red.service.search.v1.server.service.IndexDeleteService; +import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService; import com.iqser.red.service.search.v1.server.service.IndexInformationService; import com.iqser.red.service.search.v1.server.service.TextStorageService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService; -import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -54,6 +54,7 @@ public class IndexingMessageReceiver { private final RabbitTemplate rabbitTemplate; private final IndexDeleteService indexDeleteService; private final IndexInformationService indexInformationService; + private final IndexDocumentConverterService indexDocumentConverterService; @SneakyThrows @@ -88,12 +89,14 @@ public class IndexingMessageReceiver { case UPDATE: fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); - documentUpdateService.updateDocument(indexRequest.getFileId(), - fileStatus.getAssignee(), + + var indexUpdateDocument = indexDocumentConverterService.convertUpdateDocument(fileStatus.getAssignee(), dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, fileStatus.getWorkflowStatus().name(), fileStatus.getFileAttributes()); + + documentUpdateService.updateDocument(indexRequest.getFileId(), indexUpdateDocument); log.info("Successfully updated {}", indexRequest); break; @@ -152,7 +155,8 @@ public class IndexingMessageReceiver { fileStatusProcessingUpdateClient.indexing(dossier.getId(), file.getId()); Text text = textStorageService.getText(dossier.getId(), file.getId()); - documentIndexService.indexDocument(dossier.getDossierTemplateId(), + + var indexDocument = indexDocumentConverterService.convert(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), @@ -162,6 +166,8 @@ public class IndexingMessageReceiver { dossier.getArchivedTime() != null, file.getWorkflowStatus(), file.getFileAttributes()); + + documentIndexService.indexDocument(indexDocument); fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId()); log.info("Successfully indexed dossier {} file {}", dossier.getId(), file.getId()); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java deleted file mode 100644 index c1a8e0b..0000000 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.iqser.red.service.search.v1.server.service; - -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; - -import java.io.IOException; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.opensearch.action.index.IndexRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.common.xcontent.XContentType; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.model.IndexDocument; -import com.iqser.red.service.search.v1.server.model.IndexFileAttribute; -import com.iqser.red.service.search.v1.server.model.IndexSection; -import com.iqser.red.service.search.v1.server.model.SectionArea; -import com.iqser.red.service.search.v1.server.model.SectionText; -import com.iqser.red.service.search.v1.server.model.Text; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; - -import io.micrometer.core.annotation.Timed; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class DocumentIndexService { - - private final ElasticsearchClient client; - private final ElasticsearchSettings settings; - private final ObjectMapper objectMapper; - - - @Timed("redactmanager_indexDocument") - public void indexDocument(String dossierTemplateId, - String dossierId, - String fileId, - String filename, - Text text, - String assignee, - boolean deleted, - boolean archived, - WorkflowStatus workflowStatus, - Map fileAttributes) { - - IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id(fileId); - indexRequest.setRefreshPolicy(settings.getRefreshPolicy()); - indexRequest.source(toJson(convert(dossierTemplateId, dossierId, fileId, filename, text, assignee, deleted, archived, workflowStatus, fileAttributes)), XContentType.JSON); - - try { - client.index(indexRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, fileId), e); - } - } - - - public String toJson(IndexDocument indexDocument) { - - try { - return objectMapper.writeValueAsString(indexDocument); - } catch (JsonProcessingException e) { - throw new IndexException(String.format(IndexException.CONTENT_TO_JSON_ERROR, indexDocument.getFileId()), e); - } - } - - - private IndexDocument convert(String dossierTemplateId, - String dossierId, - String fileId, - String filename, - Text text, - String assignee, - boolean deleted, - boolean archived, - WorkflowStatus workflowStatus, - Map fileAttributes) { - - return IndexDocument.builder() - .dossierTemplateId(dossierTemplateId) - .dossierId(dossierId) - .fileId(fileId) - .filename(filename) - .sections(text.getSectionTexts().stream().map(this::convert).collect(Collectors.toList())) - .date(OffsetDateTime.now()) - .fileAttributes(convertFileAttributes(fileAttributes)) - .assignee(assignee) - .dossierDeleted(deleted) - .dossierArchived(archived) - .workflowStatus(workflowStatus.name()) - .build(); - } - - - private List convertFileAttributes(Map fileAttributes) { - - List converted = new ArrayList<>(); - fileAttributes.entrySet().forEach(entry -> converted.add(new IndexFileAttribute(entry.getKey(), entry.getValue()))); - return converted; - } - - - private IndexSection convert(SectionText sectionText) { - - return IndexSection.builder() - .sectionNumber(sectionText.getSectionNumber()) - .text(sectionText.getText()) - .pages(sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet())) - .headline(sectionText.getHeadline()) - .build(); - } - -} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java deleted file mode 100644 index 58c2f05..0000000 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.iqser.red.service.search.v1.server.service; - -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.opensearch.action.update.UpdateRequest; -import org.opensearch.action.update.UpdateResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.common.xcontent.XContentType; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; -import com.iqser.red.service.search.v1.server.model.IndexFileAttribute; - -import io.micrometer.core.annotation.Timed; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; - -@Service -@RequiredArgsConstructor -public class DocumentUpdateService { - - private final ObjectMapper objectMapper; - private final ElasticsearchClient client; - - - @SneakyThrows - @Timed("redactmanager_updateDocument") - public void updateDocument(String fileId, String assignee, boolean deleted, boolean archived, String workflowStatus, Map fileAttributes) { - - var indexUpdateRequest = IndexDocumentUpdate.builder() - .assignee(assignee) - .dossierDeleted(deleted) - .dossierArchived(archived) - .workflowStatus(workflowStatus) - .fileAttributes(convertFileAttributes(fileAttributes)) - .build(); - - UpdateRequest request = new UpdateRequest(INDEX_NAME, fileId); - request.doc(objectMapper.writeValueAsString(indexUpdateRequest), XContentType.JSON); - - UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT); - if (updateResponse.status().getStatus() < 200 || updateResponse.status().getStatus() > 204) { - throw new IllegalStateException("Document could not be updated. HTTP error " + updateResponse.status().getStatus()); - } - - } - - - private List convertFileAttributes(Map fileAttributes) { - - List converted = new ArrayList<>(); - if (fileAttributes != null && !fileAttributes.isEmpty()) { - fileAttributes.forEach((key, value) -> converted.add(new IndexFileAttribute(key, value))); - } - return converted; - } - -} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java deleted file mode 100644 index 8fa2122..0000000 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.iqser.red.service.search.v1.server.service; - -import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; - -import java.io.IOException; - -import org.opensearch.client.RequestOptions; -import org.opensearch.client.indices.CreateIndexRequest; -import org.opensearch.client.indices.CreateIndexResponse; -import org.opensearch.client.indices.GetIndexRequest; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentType; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import com.iqser.red.service.search.v1.server.utils.ResourceLoader; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -public class IndexCreatorService { - - public static final String INDEX_NAME = "redaction"; - - private final ElasticsearchClient client; - private final ElasticsearchSettings settings; - - - public IndexCreatorService(ElasticsearchClient client, ElasticsearchSettings settings) { - - this.client = client; - this.settings = settings; - - if (!indexExists()) { - createIndex(); - } - } - - - public void createIndex() { - - String indexMapping = ResourceLoader.load("index/mapping.json"); - String indexSettings = ResourceLoader.load("index/settings.json"); - - Settings.Builder settingsBuilder = Settings.builder() - .loadFromSource(indexSettings, XContentType.JSON) - .put("number_of_shards", settings.getNumberOfShards()) - .put("number_of_replicas", settings.getNumberOfReplicas()) - .put("index.mapping.nested_objects.limit", settings.getNumberOfNestedObjectLimit()); - - CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME).settings(settingsBuilder.build()).mapping(indexMapping, XContentType.JSON); - - try { - CreateIndexResponse response = client.indices().create(createIndexRequest, RequestOptions.DEFAULT); - log.info("Successfully created index: {}", response.index()); - } catch (IOException e) { - log.error("Failed to create index.", e); - } - } - - - private boolean indexExists() { - - GetIndexRequest getIndexRequest = new GetIndexRequest(INDEX_NAME); - try { - return client.indices().exists(getIndexRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new IndexException(INDEX_EXISTS_ERROR, e); - } - } - -} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java deleted file mode 100644 index 82c69ef..0000000 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.iqser.red.service.search.v1.server.service; - -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; - -import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.indices.CloseIndexRequest; -import org.opensearch.common.unit.TimeValue; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; - -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class IndexDeleteService { - - private final ElasticsearchClient client; - - - @SneakyThrows - public void closeIndex() { - - log.info("Will close index"); - CloseIndexRequest request = new CloseIndexRequest(INDEX_NAME); - request.setTimeout(TimeValue.timeValueMinutes(2)); - AcknowledgedResponse closeIndexResponse = client.indices().close(request, RequestOptions.DEFAULT); - - if (closeIndexResponse.isAcknowledged()) { - log.info("Index is closed"); - } else { - throw new IndexException("Error while closing index"); - } - - } - - - @SneakyThrows - public void dropIndex() { - - log.info("Will drop index"); - DeleteIndexRequest request = new DeleteIndexRequest(INDEX_NAME); - request.timeout(TimeValue.timeValueMinutes(2)); - request.timeout("2m"); - AcknowledgedResponse deleteIndexResponse = client.indices().delete(request, RequestOptions.DEFAULT); - - if (deleteIndexResponse.isAcknowledged()) { - log.info("Index is dropped"); - } else { - throw new IndexException("Error while dropping index"); - } - - } - -} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDocumentConverterService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDocumentConverterService.java new file mode 100644 index 0000000..4f06795 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDocumentConverterService.java @@ -0,0 +1,84 @@ +package com.iqser.red.service.search.v1.server.service; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.model.IndexFileAttribute; +import com.iqser.red.service.search.v1.server.model.IndexSection; +import com.iqser.red.service.search.v1.server.model.SectionArea; +import com.iqser.red.service.search.v1.server.model.SectionText; +import com.iqser.red.service.search.v1.server.model.Text; + +@Service +public class IndexDocumentConverterService { + + public IndexDocument convert(String dossierTemplateId, + String dossierId, + String fileId, + String filename, + Text text, + String assignee, + boolean deleted, + boolean archived, + WorkflowStatus workflowStatus, + Map fileAttributes) { + + return IndexDocument.builder() + .dossierTemplateId(dossierTemplateId) + .dossierId(dossierId) + .fileId(fileId) + .filename(filename) + .sections(text.getSectionTexts().stream().map(this::convert).collect(Collectors.toList())) + .date(Date.from(OffsetDateTime.now().toInstant())) + .fileAttributes(convertFileAttributes(fileAttributes)) + .assignee(assignee) + .dossierDeleted(deleted) + .dossierArchived(archived) + .workflowStatus(workflowStatus.name()) + .build(); + } + + + public IndexDocumentUpdate convertUpdateDocument(String assignee, boolean deleted, boolean archived, String workflowStatus, Map fileAttributes) { + return IndexDocumentUpdate.builder() + .assignee(assignee) + .dossierDeleted(deleted) + .dossierArchived(archived) + .workflowStatus(workflowStatus) + .fileAttributes(convertFileAttributes(fileAttributes)) + .build(); + } + + + private List convertFileAttributes(Map fileAttributes) { + + List converted = new ArrayList<>(); + if (fileAttributes != null && !fileAttributes.isEmpty()) { + fileAttributes.forEach((key, value) -> converted.add(new IndexFileAttribute(key, value))); + } + return converted; + } + + private IndexSection convert(SectionText sectionText) { + + return IndexSection.builder() + .sectionNumber(sectionText.getSectionNumber()) + .text(sectionText.getText()) + .pages(sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet())) + .headline(sectionText.getHeadline()) + .build(); + } + + + + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java deleted file mode 100644 index 4638cde..0000000 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java +++ /dev/null @@ -1,313 +0,0 @@ -package com.iqser.red.service.search.v1.server.service; - -import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.lucene.search.join.ScoreMode; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.common.text.Text; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.InnerHitBuilder; -import org.opensearch.index.query.NestedQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.fetch.subphase.FetchSourceContext; -import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.search.v1.model.MatchedDocument; -import com.iqser.red.service.search.v1.model.MatchedSection; -import com.iqser.red.service.search.v1.model.SearchResult; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.model.Query; -import com.iqser.red.service.search.v1.server.utils.QueryStringConverter; - -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.util.StringUtils; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Service -@RequiredArgsConstructor -public class SearchService { - - private final ElasticsearchClient client; - - - @Timed("redactmanager_search") - public SearchResult search(String queryString, - List dossierTemplateIds, - List dossierIds, - String fileId, - String assignee, - boolean includeDeletedDossiers, - boolean includeArchivedDossiers, - String workflowStatus, - Map fileAttributes, - int page, - int pageSize, - boolean returnSections) { - - Query query = QueryStringConverter.convert(queryString); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(convertQuery(query, - dossierTemplateIds, - dossierIds, - fileId, - assignee, - includeDeletedDossiers, - includeArchivedDossiers, - workflowStatus, - fileAttributes, - returnSections)) - .from(getPageOrDefault(page) * getPageSizeOrDefault(pageSize)) - .size(getPageSizeOrDefault(pageSize)) - .fetchSource(new String[]{"dossierId", "dossierTemplateId", "dossierDeleted", "dossierArchived", "filename", "fileId", "assignee", "dossierStatus", "workflowStatus", "fileAttributes"}, - new String[]{"sections"}) - .highlighter(new HighlightBuilder().field("sections.text").field("filename").field("fileAttributes.value").highlighterType("fvh")) - .trackScores(true); - - SearchRequest request = new SearchRequest(INDEX_NAME).source(searchSourceBuilder); - - SearchResponse response = execute(request); - - return convert(response, query); - } - - - protected SearchResponse execute(SearchRequest searchRequest) { - - try { - return client.search(searchRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new IndexException(FAILED_TO_SEARCH, e); - } - } - - - private QueryBuilder convertQuery(Query query, - List dossierTemplateIds, - List dossierIds, - String fileId, - String assignee, - boolean includeDeletedDossiers, - boolean includeArchivedDossiers, - String workflowStatus, - Map fileAttributes, - boolean returnSections) { - - BoolQueryBuilder entireQuery = QueryBuilders.boolQuery(); - BoolQueryBuilder sectionsQueries = QueryBuilders.boolQuery(); - - for (String must : query.getMusts()) { - - QueryBuilder textPhraseQuery = QueryBuilders.matchPhraseQuery("sections.text", must.toLowerCase(Locale.ROOT)).queryName(must); - QueryBuilder filenamePhraseQuery = QueryBuilders.matchPhrasePrefixQuery("filename", must.toLowerCase(Locale.ROOT)).queryName("filename." + must); - QueryBuilder fileAttributesPhraseQuery = QueryBuilders.matchPhraseQuery("fileAttributes.value", must.toLowerCase(Locale.ROOT)).queryName("fileAttributes." + must); - - QueryBuilder filenameOrTextMustQuery = QueryBuilders.boolQuery().should(textPhraseQuery).should(filenamePhraseQuery).should(fileAttributesPhraseQuery); - entireQuery.must(filenameOrTextMustQuery); - sectionsQueries.should(textPhraseQuery); - } - for (String should : query.getShoulds()) { - - QueryBuilder textTermQuery = QueryBuilders.matchPhraseQuery("sections.text", should.toLowerCase(Locale.ROOT)).queryName(should); - QueryBuilder filenameTermQuery = QueryBuilders.matchPhrasePrefixQuery("filename", should.toLowerCase(Locale.ROOT)).queryName("filename." + should); - QueryBuilder fileAttributesPhraseQuery = QueryBuilders.matchPhraseQuery("fileAttributes.value", should.toLowerCase(Locale.ROOT)).queryName("fileAttributes." + should); - entireQuery.should(textTermQuery); - entireQuery.should(filenameTermQuery); - entireQuery.should(fileAttributesPhraseQuery); - sectionsQueries.should(textTermQuery); - } - - if (returnSections) { - NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery("sections", sectionsQueries, ScoreMode.Avg) - .innerHit(new InnerHitBuilder().setSize(100) - .setFetchSourceContext(new FetchSourceContext(true, - new String[]{"sections.headline", "sections.sectionNumber", "sections.pages"}, - new String[]{"sections.text"}))); - entireQuery.should(nestedQuery); - } - - BoolQueryBuilder filterQuery = QueryBuilders.boolQuery(); - - if (dossierTemplateIds != null && !dossierTemplateIds.isEmpty()) { - - var dossierTemplateIdQueryBuilder = QueryBuilders.boolQuery(); - - for (var dossierTemplateId : dossierTemplateIds) { - if (StringUtils.isNotEmpty(dossierTemplateId)) { - dossierTemplateIdQueryBuilder = dossierTemplateIdQueryBuilder.should(QueryBuilders.matchQuery("dossierTemplateId", dossierTemplateId)); - } - } - - filterQuery.must(dossierTemplateIdQueryBuilder); - } - - if (dossierIds != null && !dossierIds.isEmpty()) { - - var dossierIdQueryBuilder = QueryBuilders.boolQuery(); - - for (var dossierId : dossierIds) { - if (StringUtils.isNotEmpty(dossierId)) { - dossierIdQueryBuilder = dossierIdQueryBuilder.should(QueryBuilders.matchQuery("dossierId", dossierId)); - } - } - - filterQuery.must(dossierIdQueryBuilder); - } - - if (StringUtils.isNotEmpty(fileId)) { - filterQuery.must(QueryBuilders.matchQuery("fileId", fileId)); - } - - if (StringUtils.isNotEmpty(assignee)) { - filterQuery.must(QueryBuilders.matchQuery("assignee", assignee)); - } - - if (includeArchivedDossiers) { - filterQuery.must(QueryBuilders.termsQuery("dossierArchived", true, false)); - } else { - filterQuery.must(QueryBuilders.termsQuery("dossierArchived", false)); - } - - if (includeDeletedDossiers) { - filterQuery.must(QueryBuilders.termsQuery("dossierDeleted", true, false)); - } else { - filterQuery.must(QueryBuilders.termsQuery("dossierDeleted", false)); - } - - if (StringUtils.isNotEmpty(workflowStatus)) { - filterQuery.must(QueryBuilders.matchQuery("workflowStatus", workflowStatus)); - } - - if (fileAttributes != null && !fileAttributes.isEmpty()) { - var fileAttributesQueryBuilder = QueryBuilders.boolQuery(); - - for (var fileAttributeKey : fileAttributes.keySet()) { - if (StringUtils.isNotEmpty(fileAttributeKey)) { - fileAttributesQueryBuilder.filter(QueryBuilders.boolQuery() - .must(QueryBuilders.matchQuery("fileAttributes.name", fileAttributeKey)) - .must(QueryBuilders.matchQuery("fileAttributes.value", fileAttributes.get(fileAttributeKey)))); - } - } - - filterQuery.must(fileAttributesQueryBuilder); - } - - return QueryBuilders.boolQuery().filter(filterQuery).must(entireQuery); - } - - - private SearchResult convert(SearchResponse response, Query query) { - - return SearchResult.builder() - .matchedDocuments(Arrays.stream(response.getHits().getHits()).map(hit -> convertSearchHit(hit, query)).collect(Collectors.toList())) - .maxScore(response.getHits().getMaxScore()) - .total(response.getHits().getTotalHits().value) - .build(); - } - - - private MatchedDocument convertSearchHit(SearchHit hit, Query query) { - - Set matchesTerms = Arrays.stream(hit.getMatchedQueries()) - .map(match -> match.contains("filename.") ? match.replace("filename.", "") : match) - .map(match -> match.contains("fileAttributes.") ? match.replace("fileAttributes.", "") : match) - .collect(Collectors.toSet()); - - Set unmatchedTerms = Stream.concat(query.getMusts().stream(), query.getShoulds().stream()).filter(term -> !matchesTerms.contains(term)).collect(Collectors.toSet()); - - MatchedDocument.MatchedDocumentBuilder matchedDocumentBuilder = MatchedDocument.builder() - .score(hit.getScore()) - .dossierId((String) hit.getSourceAsMap().get("dossierId")) - .dossierTemplateId((String) hit.getSourceAsMap().get("dossierTemplateId")) - .fileId((String) hit.getSourceAsMap().get("fileId")) - .dossierStatus((String) hit.getSourceAsMap().get("dossierStatus")) - .assignee((String) hit.getSourceAsMap().get("assignee")) - .fileAttributes(convertFileAttributes(hit.getSourceAsMap().get("fileAttributes"))) - .workflowStatus((String) hit.getSourceAsMap().get("workflowStatus")) - .fileName((String) hit.getSourceAsMap().get("fileName")) - .dossierDeleted((Boolean) hit.getSourceAsMap().get("dossierDeleted")) - .dossierArchived((Boolean) hit.getSourceAsMap().get("dossierArchived")) - .highlights(hit.getHighlightFields() - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> Arrays.stream(e.getValue().getFragments()).map(Text::string).collect(Collectors.toSet())))) - .matchedTerms(matchesTerms) - .unmatchedTerms(unmatchedTerms); - - if (hit.getInnerHits() != null) { - SearchHits sectionHits = hit.getInnerHits().get("sections"); - - matchedDocumentBuilder.matchedSections(Arrays.stream(sectionHits.getHits()).map(innerHit -> convertInnerHit(innerHit)).collect(Collectors.toList())) - .containsAllMatchedSections(sectionHits.getTotalHits().value == sectionHits.getHits().length); - } - - return matchedDocumentBuilder.build(); - - } - - - private Map convertFileAttributes(Object fileAttributesSourceMap) { - - Map fileAttributes = new HashMap<>(); - - if (fileAttributesSourceMap != null) { - List> list = new ObjectMapper().convertValue(fileAttributesSourceMap, ArrayList.class); - list.forEach(r -> fileAttributes.put(r.get("name"), r.get("value"))); - } - - return fileAttributes; - } - - - private MatchedSection convertInnerHit(SearchHit hit) { - - return MatchedSection.builder() - .headline((String) hit.getSourceAsMap().get("headline")) - .sectionNumber((Integer) hit.getSourceAsMap().get("sectionNumber")) - .pages(new HashSet<>((ArrayList) hit.getSourceAsMap().get("pages"))) - .matchedTerms(Arrays.stream(hit.getMatchedQueries()).collect(Collectors.toSet())) - .build(); - } - - - private int getPageSizeOrDefault(int pageSize) { - - if (pageSize <= 0) { - return 10; - } - return pageSize; - } - - - private int getPageOrDefault(int page) { - - if (page < 0) { - return 0; - } - return page; - } - -} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java similarity index 50% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java index c90b334..e08b160 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java @@ -1,34 +1,33 @@ -package com.iqser.red.service.search.v1.server.service; +package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; import java.io.IOException; -import org.opensearch.action.delete.DeleteRequest; -import org.opensearch.client.RequestOptions; import org.springframework.stereotype.Service; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.core.DeleteRequest; import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor public class DocumentDeleteService { - private final ElasticsearchClient client; + private final EsClient client; private final ElasticsearchSettings settings; public void deleteDocument(String fileId) { - DeleteRequest request = new DeleteRequest(INDEX_NAME).id(fileId).setRefreshPolicy(settings.getRefreshPolicy()); + DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(settings.getRefreshPolicy()).build(); try { - client.delete(request, RequestOptions.DEFAULT); - } catch (IOException e) { + client.delete(request); + } catch (IOException | ElasticsearchException e) { throw new IndexException(String.format(IndexException.DOCUMENT_DELETE_ERROR, fileId), e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java new file mode 100644 index 0000000..4d1e36b --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.search.v1.server.service.elasticsearch; + +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; + +import java.io.IOException; +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DocumentIndexService { + + private final EsClient client; + private final ElasticsearchSettings settings; + + + @Timed("redactmanager_indexDocument") + public void indexDocument(IndexDocument indexDocument) { + + try { + client.index(i -> i.index(INDEX_NAME) + .id(indexDocument.getFileId()) + .refresh(settings.getRefreshPolicy()) + .document(indexDocument)); + } catch (IOException | ElasticsearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java new file mode 100644 index 0000000..9e55c48 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java @@ -0,0 +1,37 @@ +package com.iqser.red.service.search.v1.server.service.elasticsearch; + +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; + +import java.io.IOException; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +@Service +@RequiredArgsConstructor +public class DocumentUpdateService { + + private final EsClient client; + private final ElasticsearchSettings settings; + + + @SneakyThrows + @Timed("redactmanager_updateDocument") + public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { + + try { + client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(settings.getRefreshPolicy()), IndexDocumentUpdate.class); + } catch (IOException | ElasticsearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java similarity index 72% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java index effab4e..eb53d4c 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java @@ -1,6 +1,5 @@ -package com.iqser.red.service.search.v1.server.client; +package com.iqser.red.service.search.v1.server.service.elasticsearch; -import java.io.IOException; import java.util.stream.Collectors; import javax.annotation.PostConstruct; @@ -11,13 +10,16 @@ import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.opensearch.client.RestClient; -import org.opensearch.client.RestClientBuilder; -import org.opensearch.client.RestHighLevelClient; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; import lombok.RequiredArgsConstructor; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; @@ -25,7 +27,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -public class ElasticsearchClient { +public class EsClient { // Lower timeouts should be set per request. private static final int ABSURD_HIGH_TIMEOUT = 90_000_000; @@ -33,7 +35,7 @@ public class ElasticsearchClient { private final ElasticsearchSettings settings; @Delegate - private RestHighLevelClient client; + private ElasticsearchClient client; @PostConstruct @@ -54,18 +56,17 @@ public class ElasticsearchClient { builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } - client = new RestHighLevelClient(builder); + ElasticsearchTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper()); + + this.client = new ElasticsearchClient(transport); + } @PreDestroy - public void shutdown() { + public void onShutdown() { - try { - client.close(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } + client.shutdown(); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java new file mode 100644 index 0000000..efea429 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java @@ -0,0 +1,89 @@ +package com.iqser.red.service.search.v1.server.service.elasticsearch; + +import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.search.v1.server.utils.ResourceLoader; + +import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; +import co.elastic.clients.elasticsearch.indices.IndexSettings; +import co.elastic.clients.elasticsearch.indices.MappingLimitSettingsNestedObjects; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class IndexCreatorService { + + public static final String INDEX_NAME = "redaction"; + + private final EsClient client; + private final ElasticsearchSettings settings; + + + public IndexCreatorService(EsClient client, ElasticsearchSettings settings) { + + this.client = client; + this.settings = settings; + + if (!indexExists()) { + createIndex(); + } + } + + + public void createIndex() { + + try { + var response = client.indices().create(i -> i.index(INDEX_NAME).settings(createIndexSettings()).mappings(createIndexMapping())); + log.info("Successfully created index: {}", response.index()); + } catch (IOException e) { + log.error("Failed to create index.", e); + } + } + + + private boolean indexExists() { + + try { + var response = client.indices().exists(i -> i.index(INDEX_NAME)); + return response.value(); + } catch (IOException e) { + throw new IndexException(INDEX_EXISTS_ERROR, e); + } + } + + + @SneakyThrows + private TypeMapping createIndexMapping() { + + URL resource = ResourceLoader.class.getClassLoader().getResource("index/mapping.json"); + + try (InputStream is = resource.openStream()) { + return new TypeMapping.Builder().withJson(is).build(); + } + } + + + @SneakyThrows + private IndexSettings createIndexSettings() { + + URL resource = ResourceLoader.class.getClassLoader().getResource("index/settings.json"); + + try (InputStream is = resource.openStream()) { + return new IndexSettings.Builder().withJson(is) + .numberOfShards(settings.getNumberOfShards()) + .numberOfReplicas(settings.getNumberOfReplicas()) + .mapping(m -> m.nestedObjects(MappingLimitSettingsNestedObjects.of(a -> a.limit(settings.getNumberOfNestedObjectLimit())))) + .build(); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java new file mode 100644 index 0000000..a4bee9e --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java @@ -0,0 +1,46 @@ +package com.iqser.red.service.search.v1.server.service.elasticsearch; + +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class IndexDeleteService { + + private final EsClient client; + + + @SneakyThrows + public void closeIndex() { + + var closeIndexResponse = client.indices().close(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + if (closeIndexResponse.acknowledged()) { + log.info("Index is closed"); + } else { + throw new IndexException("Error while closing index"); + } + } + + + @SneakyThrows + public void dropIndex() { + + log.info("Will drop index"); + var deleteIndexResponse = client.indices().delete(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + + if (deleteIndexResponse.acknowledged()) { + log.info("Index is dropped"); + } else { + throw new IndexException("Error while dropping index"); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java new file mode 100644 index 0000000..97d6dfa --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java @@ -0,0 +1,333 @@ +package com.iqser.red.service.search.v1.server.service.elasticsearch; + +import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.search.v1.model.MatchedDocument; +import com.iqser.red.service.search.v1.model.MatchedSection; +import com.iqser.red.service.search.v1.model.SearchResult; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.model.Query; +import com.iqser.red.service.search.v1.server.utils.QueryStringConverter; + +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode; +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.HighlightField; +import co.elastic.clients.elasticsearch.core.search.HighlighterType; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.InnerHitsResult; +import co.elastic.clients.json.JsonData; +import io.micrometer.core.annotation.Timed; +import io.micrometer.core.instrument.util.StringUtils; +import jakarta.json.JsonObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class SearchService { + + private final EsClient client; + + + @Timed("redactmanager_search") + public SearchResult search(String queryString, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + int page, + int pageSize, + boolean returnSections) { + + Query query = QueryStringConverter.convert(queryString); + + Map highlightFieldMap = new HashMap<>(); + highlightFieldMap.put("sections.text", new HighlightField.Builder().build()); + highlightFieldMap.put("filename", new HighlightField.Builder().build()); + highlightFieldMap.put("fileAttributes.value", new HighlightField.Builder().build()); + + SearchRequest request = new SearchRequest.Builder().query(convertQuery(query, + dossierTemplateIds, + dossierIds, + fileId, + assignee, + includeDeletedDossiers, + includeArchivedDossiers, + workflowStatus, + fileAttributes, + returnSections)) + .from(getPageOrDefault(page) * getPageSizeOrDefault(pageSize)) + .size(getPageSizeOrDefault(pageSize)) + .source(s -> s.filter(f -> f.includes("dossierId", + "dossierTemplateId", + "dossierDeleted", + "dossierArchived", + "filename", + "fileId", + "assignee", + "dossierStatus", + "workflowStatus", + "fileAttributes"))) + .highlight(h -> h.type(HighlighterType.FastVector).fields(highlightFieldMap)) + .trackScores(true) + .build(); + + SearchResponse response = execute(request); + + return convert(response, query); + } + + + protected SearchResponse execute(SearchRequest searchRequest) { + + try { + return client.search(searchRequest, IndexDocument.class); + } catch (IOException e) { + throw new IndexException(FAILED_TO_SEARCH, e); + } + } + + + private co.elastic.clients.elasticsearch._types.query_dsl.Query convertQuery(Query query, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + boolean returnSections) { + + var entireQuery = QueryBuilders.bool(); + var sectionsQueries = QueryBuilders.bool(); + + for (String must : query.getMusts()) { + + var textPhraseQuery = QueryBuilders.matchPhrase(q -> q.field("sections.text").query(must.toLowerCase(Locale.ROOT)).queryName(must)); + var filenamePhraseQuery = QueryBuilders.matchPhrasePrefix(q -> q.field("filename").query(must.toLowerCase(Locale.ROOT)).queryName("filename." + must)); + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase(q -> q.field("fileAttributes.value") + .query(must.toLowerCase(Locale.ROOT)) + .queryName("fileAttributes." + must)); + + var filenameOrTextMustQuery = QueryBuilders.bool().should(textPhraseQuery).should(filenamePhraseQuery).should(fileAttributesPhraseQuery).build()._toQuery(); + entireQuery.must(filenameOrTextMustQuery); + sectionsQueries.should(textPhraseQuery); + } + for (String should : query.getShoulds()) { + + var textTermQuery = QueryBuilders.matchPhrase(q -> q.field("sections.text").query(should.toLowerCase(Locale.ROOT)).queryName(should)); + var filenameTermQuery = QueryBuilders.matchPhrasePrefix(q -> q.field("filename").query(should.toLowerCase(Locale.ROOT)).queryName("filename." + should)); + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase(q -> q.field("fileAttributes.value") + .query(should.toLowerCase(Locale.ROOT)) + .queryName("fileAttributes." + should)); + entireQuery.should(textTermQuery); + entireQuery.should(filenameTermQuery); + entireQuery.should(fileAttributesPhraseQuery); + sectionsQueries.should(textTermQuery); + } + + if (returnSections) { + var nestedQuery = QueryBuilders.nested(n -> n.scoreMode(ChildScoreMode.Avg) + .queryName("sections") + .query(sectionsQueries.build()._toQuery()) + .path("sections") + .innerHits(i -> i.size(100))); + entireQuery.should(nestedQuery); + } + + var filterQuery = QueryBuilders.bool(); + + if (dossierTemplateIds != null && !dossierTemplateIds.isEmpty()) { + + var dossierTemplateIdQueryBuilder = QueryBuilders.bool(); + + for (var dossierTemplateId : dossierTemplateIds) { + if (StringUtils.isNotEmpty(dossierTemplateId)) { + dossierTemplateIdQueryBuilder = dossierTemplateIdQueryBuilder.should(QueryBuilders.match(q -> q.field("dossierTemplateId").query(dossierTemplateId))); + } + } + + filterQuery.must(dossierTemplateIdQueryBuilder.build()._toQuery()); + } + + if (dossierIds != null && !dossierIds.isEmpty()) { + + var dossierIdQueryBuilder = QueryBuilders.bool(); + + for (var dossierId : dossierIds) { + if (StringUtils.isNotEmpty(dossierId)) { + dossierIdQueryBuilder = dossierIdQueryBuilder.should(QueryBuilders.match(q -> q.field("dossierId").query(dossierId))); + } + } + + filterQuery.must(dossierIdQueryBuilder.build()._toQuery()); + } + + if (StringUtils.isNotEmpty(fileId)) { + filterQuery.must(QueryBuilders.match(q -> q.field("fileId").query(fileId))); + } + + if (StringUtils.isNotEmpty(assignee)) { + filterQuery.must(QueryBuilders.match(q -> q.field("assignee").query(assignee))); + } + + if (includeArchivedDossiers) { + filterQuery.must(QueryBuilders.terms(q -> q.field("dossierArchived") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))))); + } else { + filterQuery.must(QueryBuilders.terms(q -> q.field("dossierArchived").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))))); + } + + if (includeDeletedDossiers) { + filterQuery.must(QueryBuilders.terms(q -> q.field("dossierDeleted") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))))); + } else { + filterQuery.must(QueryBuilders.terms(q -> q.field("dossierDeleted").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))))); + } + + if (StringUtils.isNotEmpty(workflowStatus)) { + filterQuery.must(QueryBuilders.match(q -> q.field("workflowStatus").query(workflowStatus))); + } + + if (fileAttributes != null && !fileAttributes.isEmpty()) { + var fileAttributesQueryBuilder = QueryBuilders.bool(); + + for (var fileAttributeKey : fileAttributes.keySet()) { + if (StringUtils.isNotEmpty(fileAttributeKey)) { + fileAttributesQueryBuilder.filter(List.of(QueryBuilders.bool() + .must(QueryBuilders.match(q -> q.field("fileAttributes.name").query(fileAttributeKey))) + .must(QueryBuilders.match(q -> q.field("fileAttributes.value").query(fileAttributes.get(fileAttributeKey)))) + .build() + ._toQuery())); + } + } + + filterQuery.must(fileAttributesQueryBuilder.build()._toQuery()); + } + + return QueryBuilders.bool().filter(filterQuery.build()._toQuery()).must(entireQuery.build()._toQuery()).build()._toQuery(); + } + + + private SearchResult convert(SearchResponse response, Query query) { + + List hits = response.hits().hits(); + + return SearchResult.builder() + .matchedDocuments(hits.stream().map(hit -> convertSearchHit((Hit) hit, query)).collect(Collectors.toList())) + .maxScore(response.hits().maxScore().floatValue()) + .total(response.hits().total().value()) + .build(); + } + + + private MatchedDocument convertSearchHit(Hit hit, Query query) { + + List m = hit.matchedQueries(); + + Set matchesTerms = m.stream() + .map(match -> match.contains("filename.") ? match.replace("filename.", "") : match) + .map(match -> match.contains("fileAttributes.") ? match.replace("fileAttributes.", "") : match) + .collect(Collectors.toSet()); + + Set unmatchedTerms = Stream.concat(query.getMusts().stream(), query.getShoulds().stream()).filter(term -> !matchesTerms.contains(term)).collect(Collectors.toSet()); + + IndexDocument indexDocument = (IndexDocument) hit.source(); + + MatchedDocument.MatchedDocumentBuilder matchedDocumentBuilder = MatchedDocument.builder() + .score(hit.score().floatValue()) + .dossierId(indexDocument.getDossierId()) + .dossierTemplateId(indexDocument.getDossierTemplateId()) + .fileId(indexDocument.getFileId()) + .assignee(indexDocument.getAssignee()) + .fileAttributes(convertFileAttributes(indexDocument.getFileAttributes())) + .workflowStatus(indexDocument.getWorkflowStatus()) + .fileName(indexDocument.getFilename()) + .dossierDeleted(indexDocument.isDossierDeleted()) + .dossierArchived(indexDocument.isDossierArchived()) + .highlights(hit.highlight()) + .matchedTerms(matchesTerms) + .unmatchedTerms(unmatchedTerms); + + if (hit.innerHits() != null && !hit.innerHits().isEmpty()) { + InnerHitsResult sectionHits = (InnerHitsResult) hit.innerHits().get("sections"); + matchedDocumentBuilder.matchedSections(sectionHits.hits().hits().stream().map(innerHit -> convertInnerHit(innerHit)).collect(Collectors.toList())) + .containsAllMatchedSections(sectionHits.hits().total().value() == sectionHits.hits().hits().size()); + } + + return matchedDocumentBuilder.build(); + + } + + + private Map convertFileAttributes(Object fileAttributesSourceMap) { + + Map fileAttributes = new HashMap<>(); + + if (fileAttributesSourceMap != null) { + List> list = new ObjectMapper().convertValue(fileAttributesSourceMap, ArrayList.class); + list.forEach(r -> fileAttributes.put(r.get("name"), r.get("value"))); + } + + return fileAttributes; + } + + + private MatchedSection convertInnerHit(Hit hit) { + + JsonObject indexSection = hit.source().toJson().asJsonObject(); + + var jsonArray = indexSection.getJsonArray("pages"); + var pages = IntStream.range(0, jsonArray.size()).mapToObj(i -> jsonArray.getInt(i)).collect(Collectors.toSet()); + + return MatchedSection.builder() + .headline(indexSection.getString("headline")) + .sectionNumber(indexSection.getInt("sectionNumber")) + .pages(pages) + .matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet())) + .build(); + } + + + private int getPageSizeOrDefault(int pageSize) { + + if (pageSize <= 0) { + return 10; + } + return pageSize; + } + + + private int getPageOrDefault(int page) { + + if (page < 0) { + return 0; + } + return page; + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java index fef07a2..d5334be 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java @@ -3,11 +3,11 @@ package com.iqser.red.service.search.v1.server.settings; import java.util.ArrayList; import java.util.List; -import org.opensearch.action.support.WriteRequest; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import co.elastic.clients.elasticsearch._types.Refresh; import lombok.Data; @Data @@ -27,13 +27,13 @@ public class ElasticsearchSettings { private String password; - private int numberOfShards = 5; - private int numberOfReplicas = 1; + private String numberOfShards = "5"; + private String numberOfReplicas = "1"; private int numberOfNestedObjectLimit = 100000; /** * ES refresh policy for write requests to use. Used in tests to wait for completion of write requests. */ - private WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.NONE; + private Refresh refreshPolicy = Refresh.True; } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java index a0803a7..26e963f 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java @@ -13,14 +13,14 @@ import lombok.experimental.UtilityClass; @UtilityClass public class ResourceLoader { - public String load(String classpathPath) { + public InputStream load(String classpathPath) { URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath); if (resource == null) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); } - try (InputStream is = resource.openStream(); InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) { - return br.lines().collect(Collectors.joining("\n")); + try (InputStream is = resource.openStream()) { + return is; } catch (IOException e) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); } diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index cff9156..7882a59 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -20,7 +20,8 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; import com.iqser.red.service.search.v1.server.Application; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient; +import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @@ -33,7 +34,7 @@ import com.iqser.red.storage.commons.service.StorageService; @DirtiesContext public abstract class AbstractElasticsearchIntegrationTest { - public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=IMMEDIATE"; + public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=WaitFor"; static class Initializer implements ApplicationContextInitializer { @@ -64,7 +65,7 @@ public abstract class AbstractElasticsearchIntegrationTest { public static class TestConfiguration { @Bean - public IndexCreatorService indexCreationService(ElasticsearchClient elasticsearchClient, ElasticsearchSettings elasticsearchSettings) { + public IndexCreatorService indexCreationService(EsClient elasticsearchClient, ElasticsearchSettings elasticsearchSettings) { return new IndexCreatorService(elasticsearchClient, elasticsearchSettings); diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java index 98a91da..67e8a3d 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java @@ -22,6 +22,11 @@ import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.search.v1.server.client.IndexInformationClient; import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService; +import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService; import lombok.SneakyThrows; @@ -63,6 +68,9 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { @MockBean private IndexInformationService indexInformationService; + @Autowired + private IndexDocumentConverterService indexDocumentConverterService; + private final long UPDATE_TIMER = 1500; @@ -73,7 +81,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", + indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", @@ -83,7 +91,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", + indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", @@ -108,7 +116,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", + indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", @@ -148,7 +156,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", + indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", @@ -158,7 +166,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", + indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", @@ -180,14 +188,14 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); // Act & Assert 3 - documentUpdateService.updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, WorkflowStatus.NEW.name(), Map.of("F2Key", "F2Value"), 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(1); assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); // Act & Assert 4 - documentUpdateService.updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); + updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value"), 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(2); @@ -206,7 +214,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", + indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", @@ -216,7 +224,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", + indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", @@ -233,7 +241,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); // Act & Assert 2 - documentUpdateService.updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, null, null, 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(2); @@ -252,7 +260,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", + indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", @@ -262,7 +270,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", + indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", @@ -284,7 +292,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); // Act & Assert 3 - documentUpdateService.updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(1); @@ -296,7 +304,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); // Act & Assert 5 - documentUpdateService.updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(2); @@ -315,7 +323,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", + indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", @@ -325,7 +333,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", + indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", @@ -355,7 +363,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); // Act & Assert 3 - documentUpdateService.updateDocument("fileId1", "UserId2", false, true, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + updateDocument("fileId1", "UserId2", false, true, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(1); @@ -368,7 +376,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); // Act & Assert 5 - documentUpdateService.updateDocument("fileId2", "UserId2", true, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + updateDocument("fileId2", "UserId2", true, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(0); @@ -389,7 +397,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", + indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", @@ -399,7 +407,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", + indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", @@ -433,7 +441,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().size()).isEqualTo(0); // Act & Assert 6 - documentUpdateService.updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1ValueNEW")); + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1ValueNEW")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(0); @@ -444,14 +452,14 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); // Act & Assert 8 - documentUpdateService.updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1Value", "F2Key", "F2Value")); + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1Value", "F2Key", "F2Value")); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value", "F2Key", "F2Value"), 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(1); assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); // Act & Assert 9 - documentUpdateService.updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), null); + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), null); Thread.sleep(UPDATE_TIMER); result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); assertThat(result.getMatchedDocuments().size()).isEqualTo(0); @@ -467,17 +475,8 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource2 = new ClassPathResource("files/Text2.json"); Text text2 = objectMapper.readValue(textResource2.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", - "dossierId1", - "fileId1", - "Single Study - Oral (Gavage) Mouse.pdf", - text, - "UserId", - false, - false, - WorkflowStatus.NEW, - Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", + indexDocument("template1", "dossierId1", "fileId1", "Single Study - Oral (Gavage) Mouse.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", @@ -524,7 +523,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke-skywalker-42.pdf"; String searchString = "szedhsegkekhglghserlkghrsdvkerxyfdbvkrdjgh"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 1, 10, true); @@ -544,7 +543,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke-skywalker-42.pdf"; String searchString = fileName; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -566,7 +565,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke-blankwalker 42.pdf"; String searchString = fileName; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -589,7 +588,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke.pdf"; String searchString = "luke"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -611,7 +610,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "1 Abamectin_prr.pdf"; String searchString = "Abamectin"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -633,7 +632,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "1 Abamectin_prr.pdf"; String searchString = "prr"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -655,7 +654,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "1 Abamectin_prr.pdf"; String searchString = "1"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -677,7 +676,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "1 Abamectin_prr.pdf"; String searchString = "1 Abamectin_prr.pdf"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -698,7 +697,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "1 Abamectin_prr.pdf"; String searchString = "1 Abamectin"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -719,7 +718,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "1 Abamectin_prr.pdf"; String searchString = "_prr.pdf"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -738,8 +737,8 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { String fileName = "116 IDD0000261308.pdf"; String filename2 = "115 IDD0000261308.pdf"; String searchString = "\"116 IDD\""; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template1", "dossierId1", "fileId2", filename2, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId2", filename2, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -747,6 +746,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); } + @Test public void testFilenameWithNumbersMatch() throws IOException { @@ -754,7 +754,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "VV-733382.pdf"; String searchString = "733382"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -773,7 +773,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "LUKE.pdf"; String searchString = "luke"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -795,7 +795,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke.pdf"; String searchString = "luke.pdf"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -817,7 +817,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "Äpfel.pdf"; String searchString = "äpfel"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -839,7 +839,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "Äpfel.pdf"; String searchString = "äpfel.pdf"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -861,7 +861,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke-skywalker-42.pdf"; String searchString = "luke"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -883,7 +883,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke-skywalker-42.pdf"; String searchString = "luke-skywalker-42"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -906,7 +906,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { String fileName = "luke-skywalker-42.pdf"; String matchedString = fileName; String searchString = "\"" + matchedString + "\""; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -929,7 +929,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { String fileName = "luke-skywalker-42.pdf"; String matchedString = "luke"; String searchString = "\"" + matchedString + "\""; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -952,7 +952,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { String fileName = "luke-skywalker-42.pdf"; String matchedString = "luke-skywalker-42"; String searchString = "\"luke-skywalker-42\""; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -975,7 +975,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { String fileName = "luke-blankwalker 42.pdf"; String matchedString = fileName; String searchString = "\"" + matchedString + "\""; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -997,7 +997,7 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); String fileName = "luke-skywalker-42.pdf"; String searchString = "14C]-SDS-46851"; - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); // Act SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); @@ -1010,4 +1010,37 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { assertThat(StringUtils.contains(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").toArray()[0].toString(), searchString)).isTrue(); } + + private void indexDocument(String dossierTemplateId, + String dossierId, + String fileId, + String filename, + Text text, + String assignee, + boolean deleted, + boolean archived, + WorkflowStatus workflowStatus, + Map fileAttributes) { + + var indexDocument = indexDocumentConverterService.convert(dossierTemplateId, + dossierId, + fileId, + filename, + text, + assignee, + deleted, + archived, + workflowStatus, + fileAttributes); + documentIndexService.indexDocument(indexDocument); + } + + + public void updateDocument(String fileId, String assignee, boolean deleted, boolean archived, String workflowStatus, Map fileAttributes) { + + var updateDocument = indexDocumentConverterService.convertUpdateDocument(assignee, deleted, archived, workflowStatus, fileAttributes); + documentUpdateService.updateDocument(fileId, updateDocument); + + } + } From b754a1abc56de4f8dde9d8d23af9c3c2ad4a9549 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Thu, 2 Mar 2023 13:48:02 +0100 Subject: [PATCH 2/6] RED-5669: Changed tests to run with elasticsearch 8.6.0 --- .../server/service/AbstractElasticsearchIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index 7882a59..7f5704a 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -40,7 +40,8 @@ public abstract class AbstractElasticsearchIntegrationTest { public void initialize(ConfigurableApplicationContext configurableApplicationContext) { - var esContainer = new ElasticsearchContainer(DockerImageName.parse("elasticsearch:7.17.2").asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch")); + var esContainer = new ElasticsearchContainer(DockerImageName.parse("elasticsearch:8.6.0").asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch")); + esContainer.getEnvMap().put("xpack.security.enabled", "false"); esContainer.start(); String esHost = esContainer.getHttpHostAddress(); From aa22906ade744f9e5f6a395446e63a5235bc7c18 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 3 Mar 2023 14:20:55 +0100 Subject: [PATCH 3/6] RED-5669: Support both elasticsearch and opensearch --- .../search-service-server-v1/pom.xml | 26 +- .../service/search/v1/server/Application.java | 6 +- .../server/controller/SearchController.java | 2 +- .../server/queue/IndexingMessageReceiver.java | 15 +- .../server/service/DocumentDeleteService.java | 8 + .../server/service/DocumentIndexService.java | 9 + .../server/service/DocumentUpdateService.java | 9 + .../server/service/IndexCreatorService.java | 7 + .../v1/server/service/IndexDeleteService.java | 10 + .../v1/server/service/SearchService.java | 23 + ...ce.java => DocumentDeleteServiceImpl.java} | 11 +- ...ice.java => DocumentIndexServiceImpl.java} | 16 +- ...ce.java => DocumentUpdateServiceImpl.java} | 12 +- .../service/elasticsearch/EsClient.java | 9 +- ...vice.java => IndexCreatorServiceImpl.java} | 9 +- ...rvice.java => IndexDeleteServiceImpl.java} | 9 +- ...rchService.java => SearchServiceImpl.java} | 7 +- .../opensearch/DocumentDeleteServiceImpl.java | 41 + .../opensearch/DocumentIndexServiceImpl.java | 44 + .../opensearch/DocumentUpdateServiceImpl.java | 41 + .../opensearch/IndexCreatorServiceImpl.java | 108 ++ .../opensearch/IndexDeleteServiceImpl.java | 49 + .../service/opensearch/OpensearchClient.java | 73 ++ .../service/opensearch/SearchServiceImpl.java | 338 ++++++ .../settings/ElasticsearchSettings.java | 3 +- .../src/main/resources/application.yml | 3 + .../AbstractElasticsearchIntegrationTest.java | 32 +- .../AbstractOpensearchIntegrationTest.java | 69 ++ ...SearchTest.java => ElasticsearchTest.java} | 7 +- .../v1/server/service/OpensearchTest.java | 1041 +++++++++++++++++ 30 files changed, 1957 insertions(+), 80 deletions(-) create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/{DocumentDeleteService.java => DocumentDeleteServiceImpl.java} (65%) rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/{DocumentIndexService.java => DocumentIndexServiceImpl.java} (70%) rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/{DocumentUpdateService.java => DocumentUpdateServiceImpl.java} (67%) rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/{IndexCreatorService.java => IndexCreatorServiceImpl.java} (86%) rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/{IndexDeleteService.java => IndexDeleteServiceImpl.java} (76%) rename search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/{SearchService.java => SearchServiceImpl.java} (98%) create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java create mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java create mode 100644 search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java rename search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/{SearchTest.java => ElasticsearchTest.java} (98%) create mode 100644 search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java diff --git a/search-service-v1/search-service-server-v1/pom.xml b/search-service-v1/search-service-server-v1/pom.xml index dfbd9ed..c8d0ef6 100644 --- a/search-service-v1/search-service-server-v1/pom.xml +++ b/search-service-v1/search-service-server-v1/pom.xml @@ -51,17 +51,23 @@ jackson-commons - co.elastic.clients elasticsearch-java 8.6.2 - - - - + + org.opensearch.client + opensearch-rest-client + 2.6.0 + + + org.opensearch.client + opensearch-java + 2.0.0 + + jakarta.json @@ -69,10 +75,6 @@ 2.0.1 - - - - @@ -107,6 +109,12 @@ 1.16.3 test + + org.opensearch + opensearch-testcontainers + 2.0.0 + test + org.springframework.amqp spring-rabbit-test diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java index 611eb5c..d9cd90b 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.search.v1.server.client.FileStatusClient; -import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient; +import com.iqser.red.service.search.v1.server.service.opensearch.OpensearchClient; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; @@ -56,9 +56,9 @@ public class Application { @Bean @ConditionalOnMissingBean - public EsClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { + public OpensearchClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { - return new EsClient(elasticsearchSettings); + return new OpensearchClient(elasticsearchSettings); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java index 1d4a82e..c5949b7 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java @@ -6,7 +6,7 @@ import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.search.v1.model.SearchRequest; import com.iqser.red.service.search.v1.model.SearchResult; import com.iqser.red.service.search.v1.resources.SearchResource; -import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService; +import com.iqser.red.service.search.v1.server.service.SearchService; import lombok.RequiredArgsConstructor; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index 33336e5..99365f7 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -24,14 +24,14 @@ import com.iqser.red.service.search.v1.server.client.DossierClient; import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; +import com.iqser.red.service.search.v1.server.service.DocumentIndexService; +import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService; import com.iqser.red.service.search.v1.server.service.IndexInformationService; import com.iqser.red.service.search.v1.server.service.TextStorageService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -44,14 +44,15 @@ public class IndexingMessageReceiver { private final ObjectMapper objectMapper; private final TextStorageService textStorageService; - private final DocumentIndexService documentIndexService; private final FileStatusClient fileStatusClient; private final DossierClient dossierClient; private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient; + private final RabbitTemplate rabbitTemplate; + private final DocumentDeleteService documentDeleteService; private final DocumentUpdateService documentUpdateService; private final IndexCreatorService indexCreatorService; - private final RabbitTemplate rabbitTemplate; + private final DocumentIndexService documentIndexService; private final IndexDeleteService indexDeleteService; private final IndexInformationService indexInformationService; private final IndexDocumentConverterService indexDocumentConverterService; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java new file mode 100644 index 0000000..b4b0e57 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java @@ -0,0 +1,8 @@ +package com.iqser.red.service.search.v1.server.service; + +public interface DocumentDeleteService { + + void deleteDocument(String fileId); + +} + diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java new file mode 100644 index 0000000..f40b692 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.search.v1.server.service; + +import com.iqser.red.service.search.v1.server.model.IndexDocument; + +public interface DocumentIndexService { + + void indexDocument(IndexDocument indexDocument); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java new file mode 100644 index 0000000..4bf1273 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.search.v1.server.service; + +import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; + +public interface DocumentUpdateService { + + void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java new file mode 100644 index 0000000..9bb4729 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.search.v1.server.service; + +public interface IndexCreatorService { + + void createIndex(); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java new file mode 100644 index 0000000..9c2d21f --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.search.v1.server.service; + +public interface IndexDeleteService { + + void closeIndex(); + + + void dropIndex(); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java new file mode 100644 index 0000000..df5de56 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java @@ -0,0 +1,23 @@ +package com.iqser.red.service.search.v1.server.service; + +import java.util.List; +import java.util.Map; + +import com.iqser.red.service.search.v1.model.SearchResult; + +public interface SearchService { + + SearchResult search(String queryString, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + int page, + int pageSize, + boolean returnSections); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java similarity index 65% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java index e08b160..9592a1c 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java @@ -1,21 +1,25 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; import java.io.IOException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch.core.DeleteRequest; import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor -public class DocumentDeleteService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class DocumentDeleteServiceImpl implements DocumentDeleteService { private final EsClient client; private final ElasticsearchSettings settings; @@ -23,7 +27,7 @@ public class DocumentDeleteService { public void deleteDocument(String fileId) { - DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(settings.getRefreshPolicy()).build(); + DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); try { client.delete(request); @@ -34,4 +38,3 @@ public class DocumentDeleteService { } - diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java similarity index 70% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java index 4d1e36b..aa0dbdb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java @@ -1,20 +1,19 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; import java.io.IOException; -import java.util.Map; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocument; -import com.iqser.red.service.search.v1.server.model.Text; -import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService; +import com.iqser.red.service.search.v1.server.service.DocumentIndexService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -22,7 +21,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -public class DocumentIndexService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class DocumentIndexServiceImpl implements DocumentIndexService { private final EsClient client; private final ElasticsearchSettings settings; @@ -34,11 +34,11 @@ public class DocumentIndexService { try { client.index(i -> i.index(INDEX_NAME) .id(indexDocument.getFileId()) - .refresh(settings.getRefreshPolicy()) + .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) .document(indexDocument)); } catch (IOException | ElasticsearchException e) { throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e); } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java similarity index 67% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java index 9e55c48..2265f03 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java @@ -1,23 +1,27 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; import java.io.IOException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @Service @RequiredArgsConstructor -public class DocumentUpdateService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class DocumentUpdateServiceImpl implements DocumentUpdateService { private final EsClient client; private final ElasticsearchSettings settings; @@ -28,10 +32,10 @@ public class DocumentUpdateService { public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { try { - client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(settings.getRefreshPolicy()), IndexDocumentUpdate.class); + client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class); } catch (IOException | ElasticsearchException e) { throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e); } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java index eb53d4c..0664f3d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java @@ -12,11 +12,11 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.rest_client.RestClientTransport; @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") public class EsClient { // Lower timeouts should be set per request. @@ -35,7 +36,7 @@ public class EsClient { private final ElasticsearchSettings settings; @Delegate - private ElasticsearchClient client; + private co.elastic.clients.elasticsearch.ElasticsearchClient client; @PostConstruct @@ -58,7 +59,7 @@ public class EsClient { ElasticsearchTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper()); - this.client = new ElasticsearchClient(transport); + this.client = new co.elastic.clients.elasticsearch.ElasticsearchClient(transport); } @@ -69,4 +70,4 @@ public class EsClient { client.shutdown(); } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java similarity index 86% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java index efea429..50c7bab 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java @@ -6,9 +6,11 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.service.search.v1.server.utils.ResourceLoader; @@ -20,7 +22,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service -public class IndexCreatorService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class IndexCreatorServiceImpl implements IndexCreatorService { public static final String INDEX_NAME = "redaction"; @@ -28,7 +31,7 @@ public class IndexCreatorService { private final ElasticsearchSettings settings; - public IndexCreatorService(EsClient client, ElasticsearchSettings settings) { + public IndexCreatorServiceImpl(EsClient client, ElasticsearchSettings settings) { this.client = client; this.settings = settings; @@ -86,4 +89,4 @@ public class IndexCreatorService { } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java similarity index 76% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java index a4bee9e..309311d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java @@ -1,10 +1,12 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -13,7 +15,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -public class IndexDeleteService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class IndexDeleteServiceImpl implements IndexDeleteService { private final EsClient client; @@ -43,4 +46,4 @@ public class IndexDeleteService { } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java similarity index 98% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java index 97d6dfa..894b299 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java @@ -13,6 +13,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; @@ -22,6 +23,7 @@ import com.iqser.red.service.search.v1.model.SearchResult; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocument; import com.iqser.red.service.search.v1.server.model.Query; +import com.iqser.red.service.search.v1.server.service.SearchService; import com.iqser.red.service.search.v1.server.utils.QueryStringConverter; import co.elastic.clients.elasticsearch._types.FieldValue; @@ -43,7 +45,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -public class SearchService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class SearchServiceImpl implements SearchService { private final EsClient client; @@ -330,4 +333,4 @@ public class SearchService { return page; } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java new file mode 100644 index 0000000..135b916 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import java.io.IOException; + +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch._types.Refresh; +import org.opensearch.client.opensearch.core.DeleteRequest; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class DocumentDeleteServiceImpl implements DocumentDeleteService { + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + public void deleteDocument(String fileId) { + + DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); + + try { + client.delete(request); + } catch (IOException | OpenSearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_DELETE_ERROR, fileId), e); + } + } + +} + + diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java new file mode 100644 index 0000000..546f5a5 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import java.io.IOException; + +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch._types.Refresh; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.service.DocumentIndexService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class DocumentIndexServiceImpl implements DocumentIndexService { + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + @Timed("redactmanager_indexDocument") + public void indexDocument(IndexDocument indexDocument) { + + try { + client.index(i -> i.index(INDEX_NAME) + .id(indexDocument.getFileId()) + .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) + .document(indexDocument)); + } catch (IOException | OpenSearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java new file mode 100644 index 0000000..12e75e1 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import java.io.IOException; + +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch._types.Refresh; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class DocumentUpdateServiceImpl implements DocumentUpdateService { + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + @SneakyThrows + @Timed("redactmanager_updateDocument") + public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { + + try { + client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class); + } catch (IOException | OpenSearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java new file mode 100644 index 0000000..1b5dbb5 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java @@ -0,0 +1,108 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; + +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.search.v1.server.utils.ResourceLoader; + +import jakarta.json.stream.JsonParser; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class IndexCreatorServiceImpl implements IndexCreatorService { + + public static final String INDEX_NAME = "redaction"; + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + public IndexCreatorServiceImpl(OpensearchClient client, ElasticsearchSettings settings) { + + this.client = client; + this.settings = settings; + + if (!indexExists()) { + createIndex(); + } + } + + + public void createIndex() { + + try { + var response = client.indices().create(i -> i.index(INDEX_NAME).settings(createIndexSettings()).mappings(createIndexMapping())); + log.info("Successfully created index: {}", response.index()); + } catch (IOException e) { + log.error("Failed to create index.", e); + } + } + + + private boolean indexExists() { + + try { + var response = client.indices().exists(i -> i.index(INDEX_NAME)); + return response.value(); + } catch (IOException e) { + throw new IndexException(INDEX_EXISTS_ERROR, e); + } + } + + + @SneakyThrows + private TypeMapping createIndexMapping() { + + URL resource = ResourceLoader.class.getClassLoader().getResource("index/mapping.json"); + + try (InputStream is = resource.openStream()) { + + JsonpMapper mapper = client._transport().jsonpMapper(); + JsonParser parser = mapper.jsonProvider().createParser(is); + + return TypeMapping._DESERIALIZER.deserialize(parser, mapper); + } + } + + + @SneakyThrows + private IndexSettings createIndexSettings() { + + URL resource = ResourceLoader.class.getClassLoader().getResource("index/settings.json"); + + try (InputStream is = resource.openStream()) { + + JsonpMapper mapper = client._transport().jsonpMapper(); + JsonParser parser = mapper.jsonProvider().createParser(is); + + var indexSettingsFromJson = IndexSettings._DESERIALIZER.deserialize(parser, mapper); + + // It is not possible to set "index.mapping.nested_objects.limit", OpenSearch seems to not have this param. + // Hopefully they don't hava a limit for this, I was not able to find anything. + // As elasticsearch has a limit for this, and we can't set it, it seems this is the only reason for now to have both clients. + var indexSettings = new IndexSettings.Builder().index(indexSettingsFromJson.index()) + .numberOfReplicas(settings.getNumberOfReplicas()) + .numberOfShards(settings.getNumberOfShards()) + .analysis(indexSettingsFromJson.analysis()) + .build(); + + return indexSettings; + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java new file mode 100644 index 0000000..6962bb9 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java @@ -0,0 +1,49 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class IndexDeleteServiceImpl implements IndexDeleteService { + + private final OpensearchClient client; + + + @SneakyThrows + public void closeIndex() { + + var closeIndexResponse = client.indices().close(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + if (closeIndexResponse.acknowledged()) { + log.info("Index is closed"); + } else { + throw new IndexException("Error while closing index"); + } + } + + + @SneakyThrows + public void dropIndex() { + + log.info("Will drop index"); + var deleteIndexResponse = client.indices().delete(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + + if (deleteIndexResponse.acknowledged()) { + log.info("Index is dropped"); + } else { + throw new IndexException("Error while dropping index"); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java new file mode 100644 index 0000000..0ae5584 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java @@ -0,0 +1,73 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.rest_client.RestClientTransport; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class OpensearchClient { + + // Lower timeouts should be set per request. + private static final int ABSURD_HIGH_TIMEOUT = 90_000_000; + + private final ElasticsearchSettings settings; + + @Delegate + private OpenSearchClient client; + + + @PostConstruct + public void init() { + + HttpHost[] httpHost = settings.getHosts() + .stream() + .map(host -> new HttpHost(host, settings.getPort(), settings.getScheme())) + .collect(Collectors.toList()) + .toArray(new HttpHost[settings.getHosts().size()]); + + RestClientBuilder builder = RestClient.builder(httpHost) + .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT)); + + if (settings.getUsername() != null && !settings.getUsername().isEmpty()) { + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword())); + builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + var transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper()); + + this.client = new OpenSearchClient(transport); + + } + + + @PreDestroy + public void onShutdown() { + + client.shutdown(); + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java new file mode 100644 index 0000000..0cc9cde --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java @@ -0,0 +1,338 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.ChildScoreMode; +import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.BuiltinHighlighterType; +import org.opensearch.client.opensearch.core.search.HighlightField; +import org.opensearch.client.opensearch.core.search.HighlighterType; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.search.InnerHitsResult; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.search.v1.model.MatchedDocument; +import com.iqser.red.service.search.v1.model.MatchedSection; +import com.iqser.red.service.search.v1.model.SearchResult; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.model.Query; +import com.iqser.red.service.search.v1.server.service.SearchService; +import com.iqser.red.service.search.v1.server.utils.QueryStringConverter; + + +import io.micrometer.core.annotation.Timed; +import io.micrometer.core.instrument.util.StringUtils; +import jakarta.json.JsonObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class SearchServiceImpl implements SearchService { + + private final OpensearchClient client; + + + @Timed("redactmanager_search") + public SearchResult search(String queryString, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + int page, + int pageSize, + boolean returnSections) { + + Query query = QueryStringConverter.convert(queryString); + + Map highlightFieldMap = new HashMap<>(); + highlightFieldMap.put("sections.text", new HighlightField.Builder().build()); + highlightFieldMap.put("filename", new HighlightField.Builder().build()); + highlightFieldMap.put("fileAttributes.value", new HighlightField.Builder().build()); + + SearchRequest request = new SearchRequest.Builder().query(convertQuery(query, + dossierTemplateIds, + dossierIds, + fileId, + assignee, + includeDeletedDossiers, + includeArchivedDossiers, + workflowStatus, + fileAttributes, + returnSections)) + .from(getPageOrDefault(page) * getPageSizeOrDefault(pageSize)) + .size(getPageSizeOrDefault(pageSize)) + .source(s -> s.filter(f -> f.includes("dossierId", + "dossierTemplateId", + "dossierDeleted", + "dossierArchived", + "filename", + "fileId", + "assignee", + "dossierStatus", + "workflowStatus", + "fileAttributes"))) + .highlight(h -> h.type(HighlighterType.of(b -> b.builtin(BuiltinHighlighterType.FastVector))).fields(highlightFieldMap)) + .trackScores(true) + .build(); + + SearchResponse response = execute(request); + + return convert(response, query); + } + + + protected SearchResponse execute(SearchRequest searchRequest) { + + try { + return client.search(searchRequest, IndexDocument.class); + } catch (IOException e) { + throw new IndexException(FAILED_TO_SEARCH, e); + } + } + + + private org.opensearch.client.opensearch._types.query_dsl.Query convertQuery(Query query, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + boolean returnSections) { + + var entireQuery = QueryBuilders.bool(); + var sectionsQueries = QueryBuilders.bool(); + + for (String must : query.getMusts()) { + + var textPhraseQuery = QueryBuilders.matchPhrase().field("sections.text").query(must.toLowerCase(Locale.ROOT)).queryName(must).build()._toQuery(); + var filenamePhraseQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(must.toLowerCase(Locale.ROOT)).queryName("filename." + must).build()._toQuery(); + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value") + .query(must.toLowerCase(Locale.ROOT)) + .queryName("fileAttributes." + must).build()._toQuery(); + + var filenameOrTextMustQuery = QueryBuilders.bool().should(textPhraseQuery).should(filenamePhraseQuery).should(fileAttributesPhraseQuery).build()._toQuery(); + entireQuery.must(filenameOrTextMustQuery); + sectionsQueries.should(textPhraseQuery); + } + for (String should : query.getShoulds()) { + + var textTermQuery = QueryBuilders.matchPhrase().field("sections.text").query(should.toLowerCase(Locale.ROOT)).queryName(should).build()._toQuery(); + var filenameTermQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(should.toLowerCase(Locale.ROOT)).queryName("filename." + should).build()._toQuery(); + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value") + .query(should.toLowerCase(Locale.ROOT)) + .queryName("fileAttributes." + should).build()._toQuery(); + entireQuery.should(textTermQuery); + entireQuery.should(filenameTermQuery); + entireQuery.should(fileAttributesPhraseQuery); + sectionsQueries.should(textTermQuery); + } + + if (returnSections) { + var nestedQuery = QueryBuilders.nested().scoreMode(ChildScoreMode.Avg) + .queryName("sections") + .query(sectionsQueries.build()._toQuery()) + .path("sections") + .innerHits(i -> i.size(100)).build()._toQuery(); + entireQuery.should(nestedQuery); + } + + var filterQuery = QueryBuilders.bool(); + + if (dossierTemplateIds != null && !dossierTemplateIds.isEmpty()) { + + var dossierTemplateIdQueryBuilder = QueryBuilders.bool(); + + for (var dossierTemplateId : dossierTemplateIds) { + if (StringUtils.isNotEmpty(dossierTemplateId)) { + dossierTemplateIdQueryBuilder = dossierTemplateIdQueryBuilder.should(QueryBuilders.match().field("dossierTemplateId").query(q -> q.stringValue(dossierTemplateId)).build()._toQuery()); + } + } + + filterQuery.must(dossierTemplateIdQueryBuilder.build()._toQuery()); + } + + if (dossierIds != null && !dossierIds.isEmpty()) { + + var dossierIdQueryBuilder = QueryBuilders.bool(); + + for (var dossierId : dossierIds) { + if (StringUtils.isNotEmpty(dossierId)) { + dossierIdQueryBuilder = dossierIdQueryBuilder.should(QueryBuilders.match().field("dossierId").query(q -> q.stringValue(dossierId)).build()._toQuery()); + } + } + + filterQuery.must(dossierIdQueryBuilder.build()._toQuery()); + } + + if (StringUtils.isNotEmpty(fileId)) { + filterQuery.must(QueryBuilders.match().field("fileId").query(q -> q.stringValue(fileId)).build()._toQuery()); + } + + if (StringUtils.isNotEmpty(assignee)) { + filterQuery.must(QueryBuilders.match().field("assignee").query(q -> q.stringValue(assignee)).build()._toQuery()); + } + + if (includeArchivedDossiers) { + filterQuery.must(QueryBuilders.terms().field("dossierArchived") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } else { + filterQuery.must(QueryBuilders.terms().field("dossierArchived").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } + + if (includeDeletedDossiers) { + filterQuery.must(QueryBuilders.terms().field("dossierDeleted") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } else { + filterQuery.must(QueryBuilders.terms().field("dossierDeleted").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } + + if (StringUtils.isNotEmpty(workflowStatus)) { + filterQuery.must(QueryBuilders.match().field("workflowStatus").query(q -> q.stringValue(workflowStatus)).build()._toQuery()); + } + + if (fileAttributes != null && !fileAttributes.isEmpty()) { + var fileAttributesQueryBuilder = QueryBuilders.bool(); + + for (var fileAttributeKey : fileAttributes.keySet()) { + if (StringUtils.isNotEmpty(fileAttributeKey)) { + fileAttributesQueryBuilder.filter(List.of(QueryBuilders.bool() + .must(QueryBuilders.match().field("fileAttributes.name").query(q -> q.stringValue(fileAttributeKey)).build()._toQuery()) + .must(QueryBuilders.match().field("fileAttributes.value").query(q -> q.stringValue(fileAttributes.get(fileAttributeKey))).build()._toQuery()) + .build() + ._toQuery())); + } + } + + filterQuery.must(fileAttributesQueryBuilder.build()._toQuery()); + } + + return QueryBuilders.bool().filter(filterQuery.build()._toQuery()).must(entireQuery.build()._toQuery()).build()._toQuery(); + } + + + private SearchResult convert(SearchResponse response, Query query) { + + List hits = response.hits().hits(); + + return SearchResult.builder() + .matchedDocuments(hits.stream().map(hit -> convertSearchHit((Hit) hit, query)).collect(Collectors.toList())) + .maxScore(response.maxScore() == null ? 0 :response.maxScore().floatValue()) + .total(response.hits().total().value()) + .build(); + } + + + private MatchedDocument convertSearchHit(Hit hit, Query query) { + + List m = hit.matchedQueries(); + + Set matchesTerms = m.stream() + .map(match -> match.contains("filename.") ? match.replace("filename.", "") : match) + .map(match -> match.contains("fileAttributes.") ? match.replace("fileAttributes.", "") : match) + .collect(Collectors.toSet()); + + Set unmatchedTerms = Stream.concat(query.getMusts().stream(), query.getShoulds().stream()).filter(term -> !matchesTerms.contains(term)).collect(Collectors.toSet()); + + IndexDocument indexDocument = (IndexDocument) hit.source(); + + MatchedDocument.MatchedDocumentBuilder matchedDocumentBuilder = MatchedDocument.builder() + .score(hit.score().floatValue()) + .dossierId(indexDocument.getDossierId()) + .dossierTemplateId(indexDocument.getDossierTemplateId()) + .fileId(indexDocument.getFileId()) + .assignee(indexDocument.getAssignee()) + .fileAttributes(convertFileAttributes(indexDocument.getFileAttributes())) + .workflowStatus(indexDocument.getWorkflowStatus()) + .fileName(indexDocument.getFilename()) + .dossierDeleted(indexDocument.isDossierDeleted()) + .dossierArchived(indexDocument.isDossierArchived()) + .highlights(hit.highlight()) + .matchedTerms(matchesTerms) + .unmatchedTerms(unmatchedTerms); + + if (hit.innerHits() != null && !hit.innerHits().isEmpty()) { + InnerHitsResult sectionHits = (InnerHitsResult) hit.innerHits().get("sections"); + matchedDocumentBuilder.matchedSections(sectionHits.hits().hits().stream().map(innerHit -> convertInnerHit(innerHit)).collect(Collectors.toList())) + .containsAllMatchedSections(sectionHits.hits().total().value() == sectionHits.hits().hits().size()); + } + + return matchedDocumentBuilder.build(); + + } + + + private Map convertFileAttributes(Object fileAttributesSourceMap) { + + Map fileAttributes = new HashMap<>(); + + if (fileAttributesSourceMap != null) { + List> list = new ObjectMapper().convertValue(fileAttributesSourceMap, ArrayList.class); + list.forEach(r -> fileAttributes.put(r.get("name"), r.get("value"))); + } + + return fileAttributes; + } + + + private MatchedSection convertInnerHit(Hit hit) { + + JsonObject indexSection = hit.source().toJson().asJsonObject(); + + var jsonArray = indexSection.getJsonArray("pages"); + var pages = IntStream.range(0, jsonArray.size()).mapToObj(i -> jsonArray.getInt(i)).collect(Collectors.toSet()); + + return MatchedSection.builder() + .headline(indexSection.getString("headline")) + .sectionNumber(indexSection.getInt("sectionNumber")) + .pages(pages) + .matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet())) + .build(); + } + + + private int getPageSizeOrDefault(int pageSize) { + + if (pageSize <= 0) { + return 10; + } + return pageSize; + } + + + private int getPageOrDefault(int page) { + + if (page < 0) { + return 0; + } + return page; + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java index d5334be..e3e27dc 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java @@ -7,7 +7,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import co.elastic.clients.elasticsearch._types.Refresh; import lombok.Data; @Data @@ -34,6 +33,6 @@ public class ElasticsearchSettings { /** * ES refresh policy for write requests to use. Used in tests to wait for completion of write requests. */ - private Refresh refreshPolicy = Refresh.True; + private String refreshPolicy = "true"; } diff --git a/search-service-v1/search-service-server-v1/src/main/resources/application.yml b/search-service-v1/search-service-server-v1/src/main/resources/application.yml index ae02126..fbc40e5 100644 --- a/search-service-v1/search-service-server-v1/src/main/resources/application.yml +++ b/search-service-v1/search-service-server-v1/src/main/resources/application.yml @@ -34,6 +34,9 @@ management: endpoints.web.exposure.include: prometheus, health metrics.export.prometheus.enabled: ${monitoring.enabled:false} +search: + backend: elasticsearch + elasticsearch: hosts: - ${elasticsearch.cluster.hosts} diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index 7f5704a..afcc8db 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -1,6 +1,6 @@ package com.iqser.red.service.search.v1.server.service; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -15,26 +15,24 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.junit4.SpringRunner; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; import com.iqser.red.service.search.v1.server.Application; -import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @ComponentScan -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS}) +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractElasticsearchIntegrationTest.SEARCH_BACKEND}) @ContextConfiguration(initializers = {AbstractElasticsearchIntegrationTest.Initializer.class}) @EnableFeignClients(basePackageClasses = AbstractElasticsearchIntegrationTest.TestConfiguration.class) @DirtiesContext public abstract class AbstractElasticsearchIntegrationTest { - public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=WaitFor"; + public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for"; + public static final String SEARCH_BACKEND = "search.backend=elasticsearch"; static class Initializer implements ApplicationContextInitializer { @@ -45,19 +43,13 @@ public abstract class AbstractElasticsearchIntegrationTest { esContainer.start(); String esHost = esContainer.getHttpHostAddress(); -// String host = esHost.substring(0, esHost.indexOf(':')); - int port = Integer.parseInt(esHost.substring(esHost.indexOf(':') + 1)); + int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); - TestPropertyValues.of( -// "elasticsearch.cluster.hosts[0]=" + host, - "elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); + TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); } } - @Autowired - protected IndexCreatorService indexCreationService; - @Autowired protected StorageService storageService; @@ -65,14 +57,6 @@ public abstract class AbstractElasticsearchIntegrationTest { @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class}) public static class TestConfiguration { - @Bean - public IndexCreatorService indexCreationService(EsClient elasticsearchClient, ElasticsearchSettings elasticsearchSettings) { - - return new IndexCreatorService(elasticsearchClient, elasticsearchSettings); - - } - - @Bean @Primary public StorageService inmemoryStorage() { diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java new file mode 100644 index 0000000..12b3c21 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java @@ -0,0 +1,69 @@ +package com.iqser.red.service.search.v1.server.service; + +import org.junit.runner.RunWith; +import org.opensearch.testcontainers.OpensearchContainer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.testcontainers.utility.DockerImageName; + +import com.iqser.red.service.search.v1.server.Application; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; + +@ComponentScan +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractOpensearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractOpensearchIntegrationTest.SEARCH_BACKEND}) +@ContextConfiguration(initializers = {AbstractOpensearchIntegrationTest.Initializer.class}) +@EnableFeignClients(basePackageClasses = AbstractOpensearchIntegrationTest.TestConfiguration.class) +@DirtiesContext +public abstract class AbstractOpensearchIntegrationTest { + + public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for"; + public static final String SEARCH_BACKEND = "search.backend=opensearch"; + + static class Initializer implements ApplicationContextInitializer { + + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + + var esContainer = new OpensearchContainer(DockerImageName.parse("opensearchproject/opensearch:2.6.0")); + + esContainer.start(); + + String esHost = esContainer.getHttpHostAddress(); + int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); + + TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); + } + + } + + @Autowired + protected StorageService storageService; + + @Configuration + @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class}) + public static class TestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(); + } + + } + +} diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java similarity index 98% rename from search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java rename to search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java index 67e8a3d..577d25c 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java @@ -22,15 +22,10 @@ import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.search.v1.server.client.IndexInformationClient; import com.iqser.red.service.search.v1.server.model.Text; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService; import lombok.SneakyThrows; -public class SearchTest extends AbstractElasticsearchIntegrationTest { +public class ElasticsearchTest extends AbstractElasticsearchIntegrationTest { @Autowired private ObjectMapper objectMapper; diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java new file mode 100644 index 0000000..0656220 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java @@ -0,0 +1,1041 @@ +package com.iqser.red.service.search.v1.server.service; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.junit.Test; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; +import org.testcontainers.shaded.org.apache.commons.lang.StringUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.search.v1.model.MatchedDocument; +import com.iqser.red.service.search.v1.model.SearchResult; +import com.iqser.red.service.search.v1.server.client.DossierClient; +import com.iqser.red.service.search.v1.server.client.FileStatusClient; +import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; +import com.iqser.red.service.search.v1.server.client.IndexInformationClient; +import com.iqser.red.service.search.v1.server.model.Text; + +import lombok.SneakyThrows; + +public class OpensearchTest extends AbstractOpensearchIntegrationTest { + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private DocumentIndexService documentIndexService; + + @Autowired + private SearchService searchService; + + @Autowired + private DocumentDeleteService documentDeleteService; + + @Autowired + private DocumentUpdateService documentUpdateService; + + @MockBean + private FileStatusClient fileStatusClient; + + @MockBean + private FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient; + + @MockBean + private DossierClient dossierClient; + + @MockBean + private RabbitTemplate rabbitTemplate; + + @MockBean + private IndexDeleteService indexDeleteService; + + @MockBean + private IndexInformationClient indexInformationClient; + + @MockBean + private IndexInformationService indexInformationService; + + @Autowired + private IndexDocumentConverterService indexDocumentConverterService; + + private final long UPDATE_TIMER = 1500; + + + @Test + @SneakyThrows + public void testSearchFileAttribute() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("F1Value", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + } + + + @Test + @SneakyThrows + public void testSearchWithAllFilter() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value", "F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", + List.of("template1"), + List.of("dossierId1"), + "fileId1", + "UserId", + false, + false, + WorkflowStatus.NEW.name(), + Map.of("F1Key", "F1Value"), + 0, + 10, + false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + System.out.println(result); + + } + + + /* + * Index two documents and filter + */ + @Test + @SneakyThrows + public void testFilter() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.NEW.name(), null, -1, 0, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, WorkflowStatus.NEW.name(), Map.of("F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 4 + updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + } + + + /* + * Index two documents, update one and filter by assignee + */ + @Test + @SneakyThrows + public void testFilterByAssignee() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 2 + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + } + + + /* + * Index two documents, update one and filter by assignee + */ + @Test + @SneakyThrows + public void testFilterByWorkflowStatus() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.NEW.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 4 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.NEW.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 5 + updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + } + + + /* + * Index two documents, update one and filter by dossierStatus + */ + @Test + @SneakyThrows + public void testFilterByDossierStatus() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, true, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, true, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + updateDocument("fileId1", "UserId2", false, true, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 4 + result = searchService.search("S-Metolachlor", null, null, null, null, false, true, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 5 + updateDocument("fileId2", "UserId2", true, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 6 + result = searchService.search("S-Metolachlor", null, null, null, null, false, true, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + } + + + /* + * Index two documents, update one and filter by file attributes + */ + @Test + @SneakyThrows + public void testFilterByFileAttributes() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F2Key", "does-not-exist"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 4 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("Key-does-not-exist", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 5 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value", "F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 6 + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1ValueNEW")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 7 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1ValueNEW"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 8 + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1Value", "F2Key", "F2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value", "F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 9 + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), null); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + } + + + @Test + public void testTwoDocumentsAndDeleteOne() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + ClassPathResource textResource2 = new ClassPathResource("files/Text2.json"); + Text text2 = objectMapper.readValue(textResource2.getInputStream(), Text.class); + + indexDocument("template1", "dossierId1", "fileId1", "Single Study - Oral (Gavage) Mouse.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", + text2, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, List.of("dossierId1", "dossierId2"), null, null, false, false, null, null, 0, 10, false); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + // Act + documentDeleteService.deleteDocument("fileId1"); + + // Act & Assert 2 + result = searchService.search("hans klaus single", null, List.of("dossierId1", "dossierId2"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + // Act & Assert 3 + result = searchService.search("hans klaus single", null, List.of("dossierId3", "dossierId4"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 4 + result = searchService.search("hans klaus single", List.of("template1", "template2"), null, null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + } + + + /* + * SearchService will not find any result + */ + @Test + public void testWithoutResult() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "szedhsegkekhglghserlkghrsdvkerxyfdbvkrdjgh"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 1, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + } + + + /* + * Filename contains hyphen and searchString contains complete filename + */ + @Test + public void testFilenameWithHyphenWholeFilenameSearch() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = fileName; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen & blank and searchString contains complete filename + */ + @Test + public void testFilenameWithHyphenAndBlankContainsFilenameSearch() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-blankwalker 42.pdf"; + String searchString = fileName; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains("luke-blankwalker")).isTrue(); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains("42.pdf")).isTrue(); + } + + + /* + * Filename contains only one word and searchString contains complete filename without ending + */ + @Test + public void testFilenameWithOneWord() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke.pdf"; + String searchString = "luke"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains one word + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchOneWord() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "Abamectin"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains last word + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchLastWord() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "prr"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains number + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchNumber() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "1"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains whole fileName + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchCompleteFileName() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "1 Abamectin_prr.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + /* + * Filename contains number, blank, underscore and searchString contains part of fileName + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchPartOfFileNameSearchNumber() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "1 Abamectin"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + /* + * Filename contains number, blank, underscore and searchString contains part of fileName + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchLastPartOfFileName() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "_prr.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + @Test + public void testFilenameWithNumbersExactMatch() throws IOException { + + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "116 IDD0000261308.pdf"; + String filename2 = "115 IDD0000261308.pdf"; + String searchString = "\"116 IDD\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId2", filename2, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + @Test + public void testFilenameWithNumbersMatch() throws IOException { + + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "VV-733382.pdf"; + String searchString = "733382"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + /* + * Filename contains only one uppercase word and searchString contains complete filename without ending + */ + @Test + public void testFilenameWithOneWordUppercase() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "LUKE.pdf"; + String searchString = "luke"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains only one word and searchString contains complete filename with ending + */ + @Test + public void testFilenameWithOneWordAndEnding() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke.pdf"; + String searchString = "luke.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains only one word with umlaut and searchString contains complete filename without ending + */ + @Test + public void testFilenameWithOneWordWithUmlaut() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "Äpfel.pdf"; + String searchString = "äpfel"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * FFilename contains only one word with umlaut and searchString contains complete filename with ending + */ + @Test + public void testFilenameWithOneWordWithUmlautAndEnding() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "Äpfel.pdf"; + String searchString = "äpfel.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains one word from filename + */ + @Test + public void testFilenameWithHyphenOneWordSearch() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "luke"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains filename without file ending + */ + @Test + public void testFilenameWithHyphenWithoutFileEnding() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "luke-skywalker-42"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains complete filename within quotes + */ + @Test + public void testFilenameWithHyphenWholeFilenameSearchWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String matchedString = fileName; + String searchString = "\"" + matchedString + "\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains part of filename within quotes + */ + @Test + public void testFilenameWithHyphenOneWordSearchWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String matchedString = "luke"; + String searchString = "\"" + matchedString + "\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains filename without file ending within quotes + */ + @Test + public void testFilenameWithHyphenWithoutFileEndingWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String matchedString = "luke-skywalker-42"; + String searchString = "\"luke-skywalker-42\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Filename contains hyphen & blank and searchString contains complete filename within quotes + */ + @Test + public void testFilenameWithHyphenAndBlankWholeFilenameSearchWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-blankwalker 42.pdf"; + String matchedString = fileName; + String searchString = "\"" + matchedString + "\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Search string with hyphen + */ + @Test + public void testSearchStringWithHyphen() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "14C]-SDS-46851"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + assertThat(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").size()).isGreaterThan(0); + assertThat(StringUtils.contains(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").toArray()[0].toString(), searchString)).isTrue(); + } + + + private void indexDocument(String dossierTemplateId, + String dossierId, + String fileId, + String filename, + Text text, + String assignee, + boolean deleted, + boolean archived, + WorkflowStatus workflowStatus, + Map fileAttributes) { + + var indexDocument = indexDocumentConverterService.convert(dossierTemplateId, + dossierId, + fileId, + filename, + text, + assignee, + deleted, + archived, + workflowStatus, + fileAttributes); + documentIndexService.indexDocument(indexDocument); + } + + + public void updateDocument(String fileId, String assignee, boolean deleted, boolean archived, String workflowStatus, Map fileAttributes) { + + var updateDocument = indexDocumentConverterService.convertUpdateDocument(assignee, deleted, archived, workflowStatus, fileAttributes); + documentUpdateService.updateDocument(fileId, updateDocument); + + } + +} From 426ca8f133874ffefdeb332e51ad3f766604c770 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 3 Mar 2023 14:57:23 +0100 Subject: [PATCH 4/6] RED-5669: Removed unneeded code --- .../service/search/v1/server/Application.java | 23 --------------- .../IndexCreatorServiceImpl.java | 2 +- .../opensearch/IndexCreatorServiceImpl.java | 2 +- .../v1/server/utils/ResourceLoader.java | 29 ------------------- 4 files changed, 2 insertions(+), 54 deletions(-) delete mode 100644 search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java index d9cd90b..6e0505c 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java @@ -1,9 +1,5 @@ package com.iqser.red.service.search.v1.server; -import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; - -import java.util.TimeZone; - import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -13,12 +9,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.service.opensearch.OpensearchClient; @@ -40,20 +31,6 @@ public class Application { } - @Bean - @Primary - public ObjectMapper objectMapper() { - - return new ObjectMapper().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) - .enable(SerializationFeature.WRITE_DATES_WITH_ZONE_ID) - .disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE) - .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) - .registerModule(new JavaTimeModule()) - .setTimeZone(TimeZone.getDefault()) - .findAndRegisterModules(); - } - - @Bean @ConditionalOnMissingBean public OpensearchClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java index 50c7bab..13baedb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java @@ -7,12 +7,12 @@ import java.io.InputStream; import java.net.URL; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.io.ResourceLoader; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.service.IndexCreatorService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import com.iqser.red.service.search.v1.server.utils.ResourceLoader; import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; import co.elastic.clients.elasticsearch.indices.IndexSettings; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java index 1b5dbb5..f63178e 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java @@ -10,12 +10,12 @@ import org.opensearch.client.json.JsonpMapper; import org.opensearch.client.opensearch._types.mapping.TypeMapping; import org.opensearch.client.opensearch.indices.IndexSettings; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.io.ResourceLoader; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.service.IndexCreatorService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import com.iqser.red.service.search.v1.server.utils.ResourceLoader; import jakarta.json.stream.JsonParser; import lombok.SneakyThrows; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java deleted file mode 100644 index 26e963f..0000000 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.iqser.red.service.search.v1.server.utils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.stream.Collectors; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class ResourceLoader { - - public InputStream load(String classpathPath) { - - URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath); - if (resource == null) { - throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); - } - try (InputStream is = resource.openStream()) { - return is; - } catch (IOException e) { - throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); - } - } - -} From 2f91ccf538ac390809992db722b618148b88c570 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 3 Mar 2023 16:09:59 +0100 Subject: [PATCH 5/6] RED-5669: Fixed pr findings --- .../v1/server/exception/IndexException.java | 30 +++++++++++++++---- .../server/service/DocumentIndexService.java | 1 + .../DocumentDeleteServiceImpl.java | 2 +- .../DocumentIndexServiceImpl.java | 2 +- .../DocumentUpdateServiceImpl.java | 2 +- .../IndexCreatorServiceImpl.java | 4 +-- .../elasticsearch/SearchServiceImpl.java | 4 +-- .../opensearch/DocumentDeleteServiceImpl.java | 2 +- .../opensearch/DocumentIndexServiceImpl.java | 7 ++--- .../opensearch/DocumentUpdateServiceImpl.java | 5 ++-- .../opensearch/IndexCreatorServiceImpl.java | 4 +-- .../service/opensearch/SearchServiceImpl.java | 5 +--- .../AbstractElasticsearchIntegrationTest.java | 4 ++- 13 files changed, 41 insertions(+), 31 deletions(-) diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java index 5b9a5f8..58401ae 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java @@ -2,13 +2,31 @@ package com.iqser.red.service.search.v1.server.exception; public class IndexException extends RuntimeException { - public static final String INDEX_EXISTS_ERROR = "Unable to check, if index exists"; - public static final String CONTENT_TO_JSON_ERROR = "Could not convert document with id '%s' to JSON!"; - public static final String DOCUMENT_INDEX_ERROR = "Error during indexing document with id '%s'"; - public static final String DOCUMENT_UPDATE_ERROR = "Error during updating document with id '%s'"; - public static final String DOCUMENT_DELETE_ERROR = "Error during deleting document with id '%s'"; - public static final String FAILED_TO_SEARCH = "Error during search"; + private static final String INDEX_EXISTS_ERROR = "Unable to check, if index exists"; + private static final String DOCUMENT_INDEX_ERROR = "Error during indexing document with id '%s'"; + private static final String DOCUMENT_UPDATE_ERROR = "Error during updating document with id '%s'"; + private static final String DOCUMENT_DELETE_ERROR = "Error during deleting document with id '%s'"; + private static final String FAILED_TO_SEARCH = "Error during search"; + public static IndexException indexExists(Throwable cause){ + return new IndexException(INDEX_EXISTS_ERROR, cause); + } + + public static IndexException documentIndexError(String fileId, Throwable cause){ + return new IndexException(String.format(INDEX_EXISTS_ERROR, fileId), cause); + } + + public static IndexException documentUpdateError(String fileId, Throwable cause){ + return new IndexException(String.format(DOCUMENT_UPDATE_ERROR, fileId), cause); + } + + public static IndexException documentDeleteError(String fileId, Throwable cause){ + return new IndexException(String.format(DOCUMENT_DELETE_ERROR, fileId), cause); + } + + public static IndexException searchFailed(Throwable cause){ + return new IndexException(FAILED_TO_SEARCH, cause); + } public IndexException(String message) { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java index f40b692..83b11f3 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java @@ -4,6 +4,7 @@ import com.iqser.red.service.search.v1.server.model.IndexDocument; public interface DocumentIndexService { + void indexDocument(IndexDocument indexDocument); } \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java index 9592a1c..a10c978 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java @@ -32,7 +32,7 @@ public class DocumentDeleteServiceImpl implements DocumentDeleteService { try { client.delete(request); } catch (IOException | ElasticsearchException e) { - throw new IndexException(String.format(IndexException.DOCUMENT_DELETE_ERROR, fileId), e); + throw IndexException.documentDeleteError(fileId, e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java index aa0dbdb..027cc91 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java @@ -37,7 +37,7 @@ public class DocumentIndexServiceImpl implements DocumentIndexService { .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) .document(indexDocument)); } catch (IOException | ElasticsearchException e) { - throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e); + throw IndexException.documentIndexError(indexDocument.getFileId(), e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java index 2265f03..832b15e 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java @@ -34,7 +34,7 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService { try { client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class); } catch (IOException | ElasticsearchException e) { - throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e); + throw IndexException.documentUpdateError(fileId, e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java index 13baedb..aab4779 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; - import java.io.IOException; import java.io.InputStream; import java.net.URL; @@ -59,7 +57,7 @@ public class IndexCreatorServiceImpl implements IndexCreatorService { var response = client.indices().exists(i -> i.index(INDEX_NAME)); return response.value(); } catch (IOException e) { - throw new IndexException(INDEX_EXISTS_ERROR, e); + throw IndexException.indexExists(e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java index 894b299..c0c2350 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; - import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -109,7 +107,7 @@ public class SearchServiceImpl implements SearchService { try { return client.search(searchRequest, IndexDocument.class); } catch (IOException e) { - throw new IndexException(FAILED_TO_SEARCH, e); + throw IndexException.searchFailed(e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java index 135b916..012eb68 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java @@ -32,7 +32,7 @@ public class DocumentDeleteServiceImpl implements DocumentDeleteService { try { client.delete(request); } catch (IOException | OpenSearchException e) { - throw new IndexException(String.format(IndexException.DOCUMENT_DELETE_ERROR, fileId), e); + throw IndexException.documentDeleteError(fileId, e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java index 546f5a5..e97697b 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java @@ -32,12 +32,9 @@ public class DocumentIndexServiceImpl implements DocumentIndexService { public void indexDocument(IndexDocument indexDocument) { try { - client.index(i -> i.index(INDEX_NAME) - .id(indexDocument.getFileId()) - .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) - .document(indexDocument)); + client.index(i -> i.index(INDEX_NAME).id(indexDocument.getFileId()).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).document(indexDocument)); } catch (IOException | OpenSearchException e) { - throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e); + throw IndexException.documentIndexError(indexDocument.getFileId(), e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java index 12e75e1..3861c18 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java @@ -32,9 +32,10 @@ public class DocumentUpdateServiceImpl implements DocumentUpdateService { public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { try { - client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class); + client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), + IndexDocumentUpdate.class); } catch (IOException | OpenSearchException e) { - throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e); + throw IndexException.documentUpdateError(fileId, e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java index f63178e..ba099ee 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.opensearch; -import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; - import java.io.IOException; import java.io.InputStream; import java.net.URL; @@ -60,7 +58,7 @@ public class IndexCreatorServiceImpl implements IndexCreatorService { var response = client.indices().exists(i -> i.index(INDEX_NAME)); return response.value(); } catch (IOException e) { - throw new IndexException(INDEX_EXISTS_ERROR, e); + throw IndexException.indexExists(e); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java index 0cc9cde..8bb2b6b 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.service.opensearch; -import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; - import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -37,7 +35,6 @@ import com.iqser.red.service.search.v1.server.model.Query; import com.iqser.red.service.search.v1.server.service.SearchService; import com.iqser.red.service.search.v1.server.utils.QueryStringConverter; - import io.micrometer.core.annotation.Timed; import io.micrometer.core.instrument.util.StringUtils; import jakarta.json.JsonObject; @@ -111,7 +108,7 @@ public class SearchServiceImpl implements SearchService { try { return client.search(searchRequest, IndexDocument.class); } catch (IOException e) { - throw new IndexException(FAILED_TO_SEARCH, e); + throw IndexException.searchFailed(e); } } diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index afcc8db..c73141e 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -1,5 +1,6 @@ package com.iqser.red.service.search.v1.server.service; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -15,6 +16,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit4.SpringRunner; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; @@ -24,7 +26,7 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @ComponentScan -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractElasticsearchIntegrationTest.SEARCH_BACKEND}) @ContextConfiguration(initializers = {AbstractElasticsearchIntegrationTest.Initializer.class}) @EnableFeignClients(basePackageClasses = AbstractElasticsearchIntegrationTest.TestConfiguration.class) From 0deb73e31878f707ae090baf040843c25e372a09 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Mon, 6 Mar 2023 09:15:41 +0100 Subject: [PATCH 6/6] RED-5669: Fixed pr findings 2 --- .../service/search/v1/server/exception/IndexException.java | 2 +- .../service/AbstractElasticsearchIntegrationTest.java | 2 -- .../server/service/AbstractOpensearchIntegrationTest.java | 6 +++--- .../service/search/v1/server/service/OpensearchTest.java | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java index 58401ae..7313f14 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/exception/IndexException.java @@ -13,7 +13,7 @@ public class IndexException extends RuntimeException { } public static IndexException documentIndexError(String fileId, Throwable cause){ - return new IndexException(String.format(INDEX_EXISTS_ERROR, fileId), cause); + return new IndexException(String.format(DOCUMENT_INDEX_ERROR, fileId), cause); } public static IndexException documentUpdateError(String fileId, Throwable cause){ diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index c73141e..152f75c 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -1,7 +1,6 @@ package com.iqser.red.service.search.v1.server.service; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -17,7 +16,6 @@ import org.springframework.context.annotation.Primary; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; -import org.springframework.test.context.junit4.SpringRunner; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java index 12b3c21..f4700d8 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java @@ -1,6 +1,6 @@ package com.iqser.red.service.search.v1.server.service; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.testcontainers.OpensearchContainer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -16,7 +16,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.testcontainers.utility.DockerImageName; import com.iqser.red.service.search.v1.server.Application; @@ -24,7 +24,7 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @ComponentScan -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractOpensearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractOpensearchIntegrationTest.SEARCH_BACKEND}) @ContextConfiguration(initializers = {AbstractOpensearchIntegrationTest.Initializer.class}) @EnableFeignClients(basePackageClasses = AbstractOpensearchIntegrationTest.TestConfiguration.class) diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java index 0656220..05293d9 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java @@ -6,7 +6,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.springframework.amqp.rabbit.core.RabbitTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean;