diff --git a/search-service-v1/search-service-server-v1/pom.xml b/search-service-v1/search-service-server-v1/pom.xml index dfbd9ed..c8d0ef6 100644 --- a/search-service-v1/search-service-server-v1/pom.xml +++ b/search-service-v1/search-service-server-v1/pom.xml @@ -51,17 +51,23 @@ jackson-commons - co.elastic.clients elasticsearch-java 8.6.2 - - - - + + org.opensearch.client + opensearch-rest-client + 2.6.0 + + + org.opensearch.client + opensearch-java + 2.0.0 + + jakarta.json @@ -69,10 +75,6 @@ 2.0.1 - - - - @@ -107,6 +109,12 @@ 1.16.3 test + + org.opensearch + opensearch-testcontainers + 2.0.0 + test + org.springframework.amqp spring-rabbit-test diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java index 611eb5c..d9cd90b 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; import com.iqser.red.service.search.v1.server.client.FileStatusClient; -import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient; +import com.iqser.red.service.search.v1.server.service.opensearch.OpensearchClient; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; @@ -56,9 +56,9 @@ public class Application { @Bean @ConditionalOnMissingBean - public EsClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { + public OpensearchClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) { - return new EsClient(elasticsearchSettings); + return new OpensearchClient(elasticsearchSettings); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java index 1d4a82e..c5949b7 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java @@ -6,7 +6,7 @@ import org.springframework.web.bind.annotation.RestController; import com.iqser.red.service.search.v1.model.SearchRequest; import com.iqser.red.service.search.v1.model.SearchResult; import com.iqser.red.service.search.v1.resources.SearchResource; -import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService; +import com.iqser.red.service.search.v1.server.service.SearchService; import lombok.RequiredArgsConstructor; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index 33336e5..99365f7 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -24,14 +24,14 @@ import com.iqser.red.service.search.v1.server.client.DossierClient; import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; +import com.iqser.red.service.search.v1.server.service.DocumentIndexService; +import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService; import com.iqser.red.service.search.v1.server.service.IndexInformationService; import com.iqser.red.service.search.v1.server.service.TextStorageService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -44,14 +44,15 @@ public class IndexingMessageReceiver { private final ObjectMapper objectMapper; private final TextStorageService textStorageService; - private final DocumentIndexService documentIndexService; private final FileStatusClient fileStatusClient; private final DossierClient dossierClient; private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient; + private final RabbitTemplate rabbitTemplate; + private final DocumentDeleteService documentDeleteService; private final DocumentUpdateService documentUpdateService; private final IndexCreatorService indexCreatorService; - private final RabbitTemplate rabbitTemplate; + private final DocumentIndexService documentIndexService; private final IndexDeleteService indexDeleteService; private final IndexInformationService indexInformationService; private final IndexDocumentConverterService indexDocumentConverterService; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java new file mode 100644 index 0000000..b4b0e57 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java @@ -0,0 +1,8 @@ +package com.iqser.red.service.search.v1.server.service; + +public interface DocumentDeleteService { + + void deleteDocument(String fileId); + +} + diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java new file mode 100644 index 0000000..f40b692 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.search.v1.server.service; + +import com.iqser.red.service.search.v1.server.model.IndexDocument; + +public interface DocumentIndexService { + + void indexDocument(IndexDocument indexDocument); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java new file mode 100644 index 0000000..4bf1273 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.search.v1.server.service; + +import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; + +public interface DocumentUpdateService { + + void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java new file mode 100644 index 0000000..9bb4729 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.search.v1.server.service; + +public interface IndexCreatorService { + + void createIndex(); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java new file mode 100644 index 0000000..9c2d21f --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.search.v1.server.service; + +public interface IndexDeleteService { + + void closeIndex(); + + + void dropIndex(); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java new file mode 100644 index 0000000..df5de56 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java @@ -0,0 +1,23 @@ +package com.iqser.red.service.search.v1.server.service; + +import java.util.List; +import java.util.Map; + +import com.iqser.red.service.search.v1.model.SearchResult; + +public interface SearchService { + + SearchResult search(String queryString, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + int page, + int pageSize, + boolean returnSections); + +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java similarity index 65% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java index e08b160..9592a1c 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java @@ -1,21 +1,25 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; import java.io.IOException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch.core.DeleteRequest; import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor -public class DocumentDeleteService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class DocumentDeleteServiceImpl implements DocumentDeleteService { private final EsClient client; private final ElasticsearchSettings settings; @@ -23,7 +27,7 @@ public class DocumentDeleteService { public void deleteDocument(String fileId) { - DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(settings.getRefreshPolicy()).build(); + DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); try { client.delete(request); @@ -34,4 +38,3 @@ public class DocumentDeleteService { } - diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java similarity index 70% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java index 4d1e36b..aa0dbdb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java @@ -1,20 +1,19 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; import java.io.IOException; -import java.util.Map; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocument; -import com.iqser.red.service.search.v1.server.model.Text; -import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService; +import com.iqser.red.service.search.v1.server.service.DocumentIndexService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -22,7 +21,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -public class DocumentIndexService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class DocumentIndexServiceImpl implements DocumentIndexService { private final EsClient client; private final ElasticsearchSettings settings; @@ -34,11 +34,11 @@ public class DocumentIndexService { try { client.index(i -> i.index(INDEX_NAME) .id(indexDocument.getFileId()) - .refresh(settings.getRefreshPolicy()) + .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) .document(indexDocument)); } catch (IOException | ElasticsearchException e) { throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e); } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java similarity index 67% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java index 9e55c48..2265f03 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java @@ -1,23 +1,27 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; import java.io.IOException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @Service @RequiredArgsConstructor -public class DocumentUpdateService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class DocumentUpdateServiceImpl implements DocumentUpdateService { private final EsClient client; private final ElasticsearchSettings settings; @@ -28,10 +32,10 @@ public class DocumentUpdateService { public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { try { - client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(settings.getRefreshPolicy()), IndexDocumentUpdate.class); + client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class); } catch (IOException | ElasticsearchException e) { throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e); } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java index eb53d4c..0664f3d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java @@ -12,11 +12,11 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.rest_client.RestClientTransport; @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") public class EsClient { // Lower timeouts should be set per request. @@ -35,7 +36,7 @@ public class EsClient { private final ElasticsearchSettings settings; @Delegate - private ElasticsearchClient client; + private co.elastic.clients.elasticsearch.ElasticsearchClient client; @PostConstruct @@ -58,7 +59,7 @@ public class EsClient { ElasticsearchTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper()); - this.client = new ElasticsearchClient(transport); + this.client = new co.elastic.clients.elasticsearch.ElasticsearchClient(transport); } @@ -69,4 +70,4 @@ public class EsClient { client.shutdown(); } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java similarity index 86% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java index efea429..50c7bab 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java @@ -6,9 +6,11 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.service.search.v1.server.utils.ResourceLoader; @@ -20,7 +22,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service -public class IndexCreatorService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class IndexCreatorServiceImpl implements IndexCreatorService { public static final String INDEX_NAME = "redaction"; @@ -28,7 +31,7 @@ public class IndexCreatorService { private final ElasticsearchSettings settings; - public IndexCreatorService(EsClient client, ElasticsearchSettings settings) { + public IndexCreatorServiceImpl(EsClient client, ElasticsearchSettings settings) { this.client = client; this.settings = settings; @@ -86,4 +89,4 @@ public class IndexCreatorService { } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java similarity index 76% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java index a4bee9e..309311d 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java @@ -1,10 +1,12 @@ package com.iqser.red.service.search.v1.server.service.elasticsearch; -import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME; +import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -13,7 +15,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -public class IndexDeleteService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class IndexDeleteServiceImpl implements IndexDeleteService { private final EsClient client; @@ -43,4 +46,4 @@ public class IndexDeleteService { } } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java similarity index 98% rename from search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java rename to search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java index 97d6dfa..894b299 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java @@ -13,6 +13,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; @@ -22,6 +23,7 @@ import com.iqser.red.service.search.v1.model.SearchResult; import com.iqser.red.service.search.v1.server.exception.IndexException; import com.iqser.red.service.search.v1.server.model.IndexDocument; import com.iqser.red.service.search.v1.server.model.Query; +import com.iqser.red.service.search.v1.server.service.SearchService; import com.iqser.red.service.search.v1.server.utils.QueryStringConverter; import co.elastic.clients.elasticsearch._types.FieldValue; @@ -43,7 +45,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @RequiredArgsConstructor -public class SearchService { +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch") +public class SearchServiceImpl implements SearchService { private final EsClient client; @@ -330,4 +333,4 @@ public class SearchService { return page; } -} +} \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java new file mode 100644 index 0000000..135b916 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import java.io.IOException; + +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch._types.Refresh; +import org.opensearch.client.opensearch.core.DeleteRequest; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.DocumentDeleteService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class DocumentDeleteServiceImpl implements DocumentDeleteService { + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + public void deleteDocument(String fileId) { + + DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build(); + + try { + client.delete(request); + } catch (IOException | OpenSearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_DELETE_ERROR, fileId), e); + } + } + +} + + diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java new file mode 100644 index 0000000..546f5a5 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import java.io.IOException; + +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch._types.Refresh; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.service.DocumentIndexService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class DocumentIndexServiceImpl implements DocumentIndexService { + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + @Timed("redactmanager_indexDocument") + public void indexDocument(IndexDocument indexDocument) { + + try { + client.index(i -> i.index(INDEX_NAME) + .id(indexDocument.getFileId()) + .refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())) + .document(indexDocument)); + } catch (IOException | OpenSearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java new file mode 100644 index 0000000..12e75e1 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java @@ -0,0 +1,41 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import java.io.IOException; + +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.opensearch.client.opensearch._types.Refresh; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; +import com.iqser.red.service.search.v1.server.service.DocumentUpdateService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class DocumentUpdateServiceImpl implements DocumentUpdateService { + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + @SneakyThrows + @Timed("redactmanager_updateDocument") + public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) { + + try { + client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class); + } catch (IOException | OpenSearchException e) { + throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java new file mode 100644 index 0000000..1b5dbb5 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java @@ -0,0 +1,108 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; + +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexCreatorService; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.search.v1.server.utils.ResourceLoader; + +import jakarta.json.stream.JsonParser; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class IndexCreatorServiceImpl implements IndexCreatorService { + + public static final String INDEX_NAME = "redaction"; + + private final OpensearchClient client; + private final ElasticsearchSettings settings; + + + public IndexCreatorServiceImpl(OpensearchClient client, ElasticsearchSettings settings) { + + this.client = client; + this.settings = settings; + + if (!indexExists()) { + createIndex(); + } + } + + + public void createIndex() { + + try { + var response = client.indices().create(i -> i.index(INDEX_NAME).settings(createIndexSettings()).mappings(createIndexMapping())); + log.info("Successfully created index: {}", response.index()); + } catch (IOException e) { + log.error("Failed to create index.", e); + } + } + + + private boolean indexExists() { + + try { + var response = client.indices().exists(i -> i.index(INDEX_NAME)); + return response.value(); + } catch (IOException e) { + throw new IndexException(INDEX_EXISTS_ERROR, e); + } + } + + + @SneakyThrows + private TypeMapping createIndexMapping() { + + URL resource = ResourceLoader.class.getClassLoader().getResource("index/mapping.json"); + + try (InputStream is = resource.openStream()) { + + JsonpMapper mapper = client._transport().jsonpMapper(); + JsonParser parser = mapper.jsonProvider().createParser(is); + + return TypeMapping._DESERIALIZER.deserialize(parser, mapper); + } + } + + + @SneakyThrows + private IndexSettings createIndexSettings() { + + URL resource = ResourceLoader.class.getClassLoader().getResource("index/settings.json"); + + try (InputStream is = resource.openStream()) { + + JsonpMapper mapper = client._transport().jsonpMapper(); + JsonParser parser = mapper.jsonProvider().createParser(is); + + var indexSettingsFromJson = IndexSettings._DESERIALIZER.deserialize(parser, mapper); + + // It is not possible to set "index.mapping.nested_objects.limit", OpenSearch seems to not have this param. + // Hopefully they don't hava a limit for this, I was not able to find anything. + // As elasticsearch has a limit for this, and we can't set it, it seems this is the only reason for now to have both clients. + var indexSettings = new IndexSettings.Builder().index(indexSettingsFromJson.index()) + .numberOfReplicas(settings.getNumberOfReplicas()) + .numberOfShards(settings.getNumberOfShards()) + .analysis(indexSettingsFromJson.analysis()) + .build(); + + return indexSettings; + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java new file mode 100644 index 0000000..6962bb9 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java @@ -0,0 +1,49 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.service.IndexDeleteService; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class IndexDeleteServiceImpl implements IndexDeleteService { + + private final OpensearchClient client; + + + @SneakyThrows + public void closeIndex() { + + var closeIndexResponse = client.indices().close(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + if (closeIndexResponse.acknowledged()) { + log.info("Index is closed"); + } else { + throw new IndexException("Error while closing index"); + } + } + + + @SneakyThrows + public void dropIndex() { + + log.info("Will drop index"); + var deleteIndexResponse = client.indices().delete(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m"))); + + if (deleteIndexResponse.acknowledged()) { + log.info("Index is dropped"); + } else { + throw new IndexException("Error while dropping index"); + } + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java new file mode 100644 index 0000000..0ae5584 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java @@ -0,0 +1,73 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.rest_client.RestClientTransport; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class OpensearchClient { + + // Lower timeouts should be set per request. + private static final int ABSURD_HIGH_TIMEOUT = 90_000_000; + + private final ElasticsearchSettings settings; + + @Delegate + private OpenSearchClient client; + + + @PostConstruct + public void init() { + + HttpHost[] httpHost = settings.getHosts() + .stream() + .map(host -> new HttpHost(host, settings.getPort(), settings.getScheme())) + .collect(Collectors.toList()) + .toArray(new HttpHost[settings.getHosts().size()]); + + RestClientBuilder builder = RestClient.builder(httpHost) + .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT)); + + if (settings.getUsername() != null && !settings.getUsername().isEmpty()) { + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword())); + builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + var transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper()); + + this.client = new OpenSearchClient(transport); + + } + + + @PreDestroy + public void onShutdown() { + + client.shutdown(); + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java new file mode 100644 index 0000000..0cc9cde --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java @@ -0,0 +1,338 @@ +package com.iqser.red.service.search.v1.server.service.opensearch; + +import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.ChildScoreMode; +import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.BuiltinHighlighterType; +import org.opensearch.client.opensearch.core.search.HighlightField; +import org.opensearch.client.opensearch.core.search.HighlighterType; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.search.InnerHitsResult; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.search.v1.model.MatchedDocument; +import com.iqser.red.service.search.v1.model.MatchedSection; +import com.iqser.red.service.search.v1.model.SearchResult; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.model.Query; +import com.iqser.red.service.search.v1.server.service.SearchService; +import com.iqser.red.service.search.v1.server.utils.QueryStringConverter; + + +import io.micrometer.core.annotation.Timed; +import io.micrometer.core.instrument.util.StringUtils; +import jakarta.json.JsonObject; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch") +public class SearchServiceImpl implements SearchService { + + private final OpensearchClient client; + + + @Timed("redactmanager_search") + public SearchResult search(String queryString, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + int page, + int pageSize, + boolean returnSections) { + + Query query = QueryStringConverter.convert(queryString); + + Map highlightFieldMap = new HashMap<>(); + highlightFieldMap.put("sections.text", new HighlightField.Builder().build()); + highlightFieldMap.put("filename", new HighlightField.Builder().build()); + highlightFieldMap.put("fileAttributes.value", new HighlightField.Builder().build()); + + SearchRequest request = new SearchRequest.Builder().query(convertQuery(query, + dossierTemplateIds, + dossierIds, + fileId, + assignee, + includeDeletedDossiers, + includeArchivedDossiers, + workflowStatus, + fileAttributes, + returnSections)) + .from(getPageOrDefault(page) * getPageSizeOrDefault(pageSize)) + .size(getPageSizeOrDefault(pageSize)) + .source(s -> s.filter(f -> f.includes("dossierId", + "dossierTemplateId", + "dossierDeleted", + "dossierArchived", + "filename", + "fileId", + "assignee", + "dossierStatus", + "workflowStatus", + "fileAttributes"))) + .highlight(h -> h.type(HighlighterType.of(b -> b.builtin(BuiltinHighlighterType.FastVector))).fields(highlightFieldMap)) + .trackScores(true) + .build(); + + SearchResponse response = execute(request); + + return convert(response, query); + } + + + protected SearchResponse execute(SearchRequest searchRequest) { + + try { + return client.search(searchRequest, IndexDocument.class); + } catch (IOException e) { + throw new IndexException(FAILED_TO_SEARCH, e); + } + } + + + private org.opensearch.client.opensearch._types.query_dsl.Query convertQuery(Query query, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + boolean returnSections) { + + var entireQuery = QueryBuilders.bool(); + var sectionsQueries = QueryBuilders.bool(); + + for (String must : query.getMusts()) { + + var textPhraseQuery = QueryBuilders.matchPhrase().field("sections.text").query(must.toLowerCase(Locale.ROOT)).queryName(must).build()._toQuery(); + var filenamePhraseQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(must.toLowerCase(Locale.ROOT)).queryName("filename." + must).build()._toQuery(); + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value") + .query(must.toLowerCase(Locale.ROOT)) + .queryName("fileAttributes." + must).build()._toQuery(); + + var filenameOrTextMustQuery = QueryBuilders.bool().should(textPhraseQuery).should(filenamePhraseQuery).should(fileAttributesPhraseQuery).build()._toQuery(); + entireQuery.must(filenameOrTextMustQuery); + sectionsQueries.should(textPhraseQuery); + } + for (String should : query.getShoulds()) { + + var textTermQuery = QueryBuilders.matchPhrase().field("sections.text").query(should.toLowerCase(Locale.ROOT)).queryName(should).build()._toQuery(); + var filenameTermQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(should.toLowerCase(Locale.ROOT)).queryName("filename." + should).build()._toQuery(); + var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value") + .query(should.toLowerCase(Locale.ROOT)) + .queryName("fileAttributes." + should).build()._toQuery(); + entireQuery.should(textTermQuery); + entireQuery.should(filenameTermQuery); + entireQuery.should(fileAttributesPhraseQuery); + sectionsQueries.should(textTermQuery); + } + + if (returnSections) { + var nestedQuery = QueryBuilders.nested().scoreMode(ChildScoreMode.Avg) + .queryName("sections") + .query(sectionsQueries.build()._toQuery()) + .path("sections") + .innerHits(i -> i.size(100)).build()._toQuery(); + entireQuery.should(nestedQuery); + } + + var filterQuery = QueryBuilders.bool(); + + if (dossierTemplateIds != null && !dossierTemplateIds.isEmpty()) { + + var dossierTemplateIdQueryBuilder = QueryBuilders.bool(); + + for (var dossierTemplateId : dossierTemplateIds) { + if (StringUtils.isNotEmpty(dossierTemplateId)) { + dossierTemplateIdQueryBuilder = dossierTemplateIdQueryBuilder.should(QueryBuilders.match().field("dossierTemplateId").query(q -> q.stringValue(dossierTemplateId)).build()._toQuery()); + } + } + + filterQuery.must(dossierTemplateIdQueryBuilder.build()._toQuery()); + } + + if (dossierIds != null && !dossierIds.isEmpty()) { + + var dossierIdQueryBuilder = QueryBuilders.bool(); + + for (var dossierId : dossierIds) { + if (StringUtils.isNotEmpty(dossierId)) { + dossierIdQueryBuilder = dossierIdQueryBuilder.should(QueryBuilders.match().field("dossierId").query(q -> q.stringValue(dossierId)).build()._toQuery()); + } + } + + filterQuery.must(dossierIdQueryBuilder.build()._toQuery()); + } + + if (StringUtils.isNotEmpty(fileId)) { + filterQuery.must(QueryBuilders.match().field("fileId").query(q -> q.stringValue(fileId)).build()._toQuery()); + } + + if (StringUtils.isNotEmpty(assignee)) { + filterQuery.must(QueryBuilders.match().field("assignee").query(q -> q.stringValue(assignee)).build()._toQuery()); + } + + if (includeArchivedDossiers) { + filterQuery.must(QueryBuilders.terms().field("dossierArchived") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } else { + filterQuery.must(QueryBuilders.terms().field("dossierArchived").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } + + if (includeDeletedDossiers) { + filterQuery.must(QueryBuilders.terms().field("dossierDeleted") + .terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } else { + filterQuery.must(QueryBuilders.terms().field("dossierDeleted").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery()); + } + + if (StringUtils.isNotEmpty(workflowStatus)) { + filterQuery.must(QueryBuilders.match().field("workflowStatus").query(q -> q.stringValue(workflowStatus)).build()._toQuery()); + } + + if (fileAttributes != null && !fileAttributes.isEmpty()) { + var fileAttributesQueryBuilder = QueryBuilders.bool(); + + for (var fileAttributeKey : fileAttributes.keySet()) { + if (StringUtils.isNotEmpty(fileAttributeKey)) { + fileAttributesQueryBuilder.filter(List.of(QueryBuilders.bool() + .must(QueryBuilders.match().field("fileAttributes.name").query(q -> q.stringValue(fileAttributeKey)).build()._toQuery()) + .must(QueryBuilders.match().field("fileAttributes.value").query(q -> q.stringValue(fileAttributes.get(fileAttributeKey))).build()._toQuery()) + .build() + ._toQuery())); + } + } + + filterQuery.must(fileAttributesQueryBuilder.build()._toQuery()); + } + + return QueryBuilders.bool().filter(filterQuery.build()._toQuery()).must(entireQuery.build()._toQuery()).build()._toQuery(); + } + + + private SearchResult convert(SearchResponse response, Query query) { + + List hits = response.hits().hits(); + + return SearchResult.builder() + .matchedDocuments(hits.stream().map(hit -> convertSearchHit((Hit) hit, query)).collect(Collectors.toList())) + .maxScore(response.maxScore() == null ? 0 :response.maxScore().floatValue()) + .total(response.hits().total().value()) + .build(); + } + + + private MatchedDocument convertSearchHit(Hit hit, Query query) { + + List m = hit.matchedQueries(); + + Set matchesTerms = m.stream() + .map(match -> match.contains("filename.") ? match.replace("filename.", "") : match) + .map(match -> match.contains("fileAttributes.") ? match.replace("fileAttributes.", "") : match) + .collect(Collectors.toSet()); + + Set unmatchedTerms = Stream.concat(query.getMusts().stream(), query.getShoulds().stream()).filter(term -> !matchesTerms.contains(term)).collect(Collectors.toSet()); + + IndexDocument indexDocument = (IndexDocument) hit.source(); + + MatchedDocument.MatchedDocumentBuilder matchedDocumentBuilder = MatchedDocument.builder() + .score(hit.score().floatValue()) + .dossierId(indexDocument.getDossierId()) + .dossierTemplateId(indexDocument.getDossierTemplateId()) + .fileId(indexDocument.getFileId()) + .assignee(indexDocument.getAssignee()) + .fileAttributes(convertFileAttributes(indexDocument.getFileAttributes())) + .workflowStatus(indexDocument.getWorkflowStatus()) + .fileName(indexDocument.getFilename()) + .dossierDeleted(indexDocument.isDossierDeleted()) + .dossierArchived(indexDocument.isDossierArchived()) + .highlights(hit.highlight()) + .matchedTerms(matchesTerms) + .unmatchedTerms(unmatchedTerms); + + if (hit.innerHits() != null && !hit.innerHits().isEmpty()) { + InnerHitsResult sectionHits = (InnerHitsResult) hit.innerHits().get("sections"); + matchedDocumentBuilder.matchedSections(sectionHits.hits().hits().stream().map(innerHit -> convertInnerHit(innerHit)).collect(Collectors.toList())) + .containsAllMatchedSections(sectionHits.hits().total().value() == sectionHits.hits().hits().size()); + } + + return matchedDocumentBuilder.build(); + + } + + + private Map convertFileAttributes(Object fileAttributesSourceMap) { + + Map fileAttributes = new HashMap<>(); + + if (fileAttributesSourceMap != null) { + List> list = new ObjectMapper().convertValue(fileAttributesSourceMap, ArrayList.class); + list.forEach(r -> fileAttributes.put(r.get("name"), r.get("value"))); + } + + return fileAttributes; + } + + + private MatchedSection convertInnerHit(Hit hit) { + + JsonObject indexSection = hit.source().toJson().asJsonObject(); + + var jsonArray = indexSection.getJsonArray("pages"); + var pages = IntStream.range(0, jsonArray.size()).mapToObj(i -> jsonArray.getInt(i)).collect(Collectors.toSet()); + + return MatchedSection.builder() + .headline(indexSection.getString("headline")) + .sectionNumber(indexSection.getInt("sectionNumber")) + .pages(pages) + .matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet())) + .build(); + } + + + private int getPageSizeOrDefault(int pageSize) { + + if (pageSize <= 0) { + return 10; + } + return pageSize; + } + + + private int getPageOrDefault(int page) { + + if (page < 0) { + return 0; + } + return page; + } + +} diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java index d5334be..e3e27dc 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java @@ -7,7 +7,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import co.elastic.clients.elasticsearch._types.Refresh; import lombok.Data; @Data @@ -34,6 +33,6 @@ public class ElasticsearchSettings { /** * ES refresh policy for write requests to use. Used in tests to wait for completion of write requests. */ - private Refresh refreshPolicy = Refresh.True; + private String refreshPolicy = "true"; } diff --git a/search-service-v1/search-service-server-v1/src/main/resources/application.yml b/search-service-v1/search-service-server-v1/src/main/resources/application.yml index ae02126..fbc40e5 100644 --- a/search-service-v1/search-service-server-v1/src/main/resources/application.yml +++ b/search-service-v1/search-service-server-v1/src/main/resources/application.yml @@ -34,6 +34,9 @@ management: endpoints.web.exposure.include: prometheus, health metrics.export.prometheus.enabled: ${monitoring.enabled:false} +search: + backend: elasticsearch + elasticsearch: hosts: - ${elasticsearch.cluster.hosts} diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index 7f5704a..afcc8db 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -1,6 +1,6 @@ package com.iqser.red.service.search.v1.server.service; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; @@ -15,26 +15,24 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.junit4.SpringRunner; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; import com.iqser.red.service.search.v1.server.Application; -import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @ComponentScan -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS}) +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractElasticsearchIntegrationTest.SEARCH_BACKEND}) @ContextConfiguration(initializers = {AbstractElasticsearchIntegrationTest.Initializer.class}) @EnableFeignClients(basePackageClasses = AbstractElasticsearchIntegrationTest.TestConfiguration.class) @DirtiesContext public abstract class AbstractElasticsearchIntegrationTest { - public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=WaitFor"; + public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for"; + public static final String SEARCH_BACKEND = "search.backend=elasticsearch"; static class Initializer implements ApplicationContextInitializer { @@ -45,19 +43,13 @@ public abstract class AbstractElasticsearchIntegrationTest { esContainer.start(); String esHost = esContainer.getHttpHostAddress(); -// String host = esHost.substring(0, esHost.indexOf(':')); - int port = Integer.parseInt(esHost.substring(esHost.indexOf(':') + 1)); + int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); - TestPropertyValues.of( -// "elasticsearch.cluster.hosts[0]=" + host, - "elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); + TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); } } - @Autowired - protected IndexCreatorService indexCreationService; - @Autowired protected StorageService storageService; @@ -65,14 +57,6 @@ public abstract class AbstractElasticsearchIntegrationTest { @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class}) public static class TestConfiguration { - @Bean - public IndexCreatorService indexCreationService(EsClient elasticsearchClient, ElasticsearchSettings elasticsearchSettings) { - - return new IndexCreatorService(elasticsearchClient, elasticsearchSettings); - - } - - @Bean @Primary public StorageService inmemoryStorage() { diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java new file mode 100644 index 0000000..12b3c21 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java @@ -0,0 +1,69 @@ +package com.iqser.red.service.search.v1.server.service; + +import org.junit.runner.RunWith; +import org.opensearch.testcontainers.OpensearchContainer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.testcontainers.utility.DockerImageName; + +import com.iqser.red.service.search.v1.server.Application; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; + +@ComponentScan +@RunWith(SpringRunner.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractOpensearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractOpensearchIntegrationTest.SEARCH_BACKEND}) +@ContextConfiguration(initializers = {AbstractOpensearchIntegrationTest.Initializer.class}) +@EnableFeignClients(basePackageClasses = AbstractOpensearchIntegrationTest.TestConfiguration.class) +@DirtiesContext +public abstract class AbstractOpensearchIntegrationTest { + + public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for"; + public static final String SEARCH_BACKEND = "search.backend=opensearch"; + + static class Initializer implements ApplicationContextInitializer { + + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + + var esContainer = new OpensearchContainer(DockerImageName.parse("opensearchproject/opensearch:2.6.0")); + + esContainer.start(); + + String esHost = esContainer.getHttpHostAddress(); + int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1)); + + TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); + } + + } + + @Autowired + protected StorageService storageService; + + @Configuration + @EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class}) + public static class TestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(); + } + + } + +} diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java similarity index 98% rename from search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java rename to search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java index 67e8a3d..577d25c 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java @@ -22,15 +22,10 @@ import com.iqser.red.service.search.v1.server.client.FileStatusClient; import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.search.v1.server.client.IndexInformationClient; import com.iqser.red.service.search.v1.server.model.Text; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService; -import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService; import lombok.SneakyThrows; -public class SearchTest extends AbstractElasticsearchIntegrationTest { +public class ElasticsearchTest extends AbstractElasticsearchIntegrationTest { @Autowired private ObjectMapper objectMapper; diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java new file mode 100644 index 0000000..0656220 --- /dev/null +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java @@ -0,0 +1,1041 @@ +package com.iqser.red.service.search.v1.server.service; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.junit.Test; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.core.io.ClassPathResource; +import org.testcontainers.shaded.org.apache.commons.lang.StringUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.search.v1.model.MatchedDocument; +import com.iqser.red.service.search.v1.model.SearchResult; +import com.iqser.red.service.search.v1.server.client.DossierClient; +import com.iqser.red.service.search.v1.server.client.FileStatusClient; +import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient; +import com.iqser.red.service.search.v1.server.client.IndexInformationClient; +import com.iqser.red.service.search.v1.server.model.Text; + +import lombok.SneakyThrows; + +public class OpensearchTest extends AbstractOpensearchIntegrationTest { + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private DocumentIndexService documentIndexService; + + @Autowired + private SearchService searchService; + + @Autowired + private DocumentDeleteService documentDeleteService; + + @Autowired + private DocumentUpdateService documentUpdateService; + + @MockBean + private FileStatusClient fileStatusClient; + + @MockBean + private FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient; + + @MockBean + private DossierClient dossierClient; + + @MockBean + private RabbitTemplate rabbitTemplate; + + @MockBean + private IndexDeleteService indexDeleteService; + + @MockBean + private IndexInformationClient indexInformationClient; + + @MockBean + private IndexInformationService indexInformationService; + + @Autowired + private IndexDocumentConverterService indexDocumentConverterService; + + private final long UPDATE_TIMER = 1500; + + + @Test + @SneakyThrows + public void testSearchFileAttribute() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("F1Value", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + } + + + @Test + @SneakyThrows + public void testSearchWithAllFilter() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value", "F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", + List.of("template1"), + List.of("dossierId1"), + "fileId1", + "UserId", + false, + false, + WorkflowStatus.NEW.name(), + Map.of("F1Key", "F1Value"), + 0, + 10, + false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + System.out.println(result); + + } + + + /* + * Index two documents and filter + */ + @Test + @SneakyThrows + public void testFilter() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.NEW.name(), null, -1, 0, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, WorkflowStatus.NEW.name(), Map.of("F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 4 + updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + } + + + /* + * Index two documents, update one and filter by assignee + */ + @Test + @SneakyThrows + public void testFilterByAssignee() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 2 + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + } + + + /* + * Index two documents, update one and filter by assignee + */ + @Test + @SneakyThrows + public void testFilterByWorkflowStatus() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.NEW.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + updateDocument("fileId1", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 4 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.NEW.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 5 + updateDocument("fileId2", "UserId2", false, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + } + + + /* + * Index two documents, update one and filter by dossierStatus + */ + @Test + @SneakyThrows + public void testFilterByDossierStatus() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, true, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, true, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + updateDocument("fileId1", "UserId2", false, true, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 4 + result = searchService.search("S-Metolachlor", null, null, null, null, false, true, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(2); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 5 + updateDocument("fileId2", "UserId2", true, false, WorkflowStatus.APPROVED.name(), Map.of("U2Key", "U2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 6 + result = searchService.search("S-Metolachlor", null, null, null, null, false, true, null, null, 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + } + + + /* + * Index two documents, update one and filter by file attributes + */ + @Test + @SneakyThrows + public void testFilterByFileAttributes() { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text2.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 2 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId2"); + + // Act & Assert 3 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F2Key", "does-not-exist"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 4 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("Key-does-not-exist", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 5 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value", "F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 6 + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1ValueNEW")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 7 + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1ValueNEW"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 8 + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1Value", "F2Key", "F2Value")); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value", "F2Key", "F2Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().stream().map(MatchedDocument::getFileId)).contains("fileId1"); + + // Act & Assert 9 + updateDocument("fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), null); + Thread.sleep(UPDATE_TIMER); + result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + } + + + @Test + public void testTwoDocumentsAndDeleteOne() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + + ClassPathResource textResource2 = new ClassPathResource("files/Text2.json"); + Text text2 = objectMapper.readValue(textResource2.getInputStream(), Text.class); + + indexDocument("template1", "dossierId1", "fileId1", "Single Study - Oral (Gavage) Mouse.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", + text2, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + + // Act & Assert 1 + SearchResult result = searchService.search("S-Metolachlor", null, List.of("dossierId1", "dossierId2"), null, null, false, false, null, null, 0, 10, false); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + // Act + documentDeleteService.deleteDocument("fileId1"); + + // Act & Assert 2 + result = searchService.search("hans klaus single", null, List.of("dossierId1", "dossierId2"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + // Act & Assert 3 + result = searchService.search("hans klaus single", null, List.of("dossierId3", "dossierId4"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + + // Act & Assert 4 + result = searchService.search("hans klaus single", List.of("template1", "template2"), null, null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + + } + + + /* + * SearchService will not find any result + */ + @Test + public void testWithoutResult() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "szedhsegkekhglghserlkghrsdvkerxyfdbvkrdjgh"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 1, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(0); + } + + + /* + * Filename contains hyphen and searchString contains complete filename + */ + @Test + public void testFilenameWithHyphenWholeFilenameSearch() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = fileName; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen & blank and searchString contains complete filename + */ + @Test + public void testFilenameWithHyphenAndBlankContainsFilenameSearch() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-blankwalker 42.pdf"; + String searchString = fileName; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains("luke-blankwalker")).isTrue(); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains("42.pdf")).isTrue(); + } + + + /* + * Filename contains only one word and searchString contains complete filename without ending + */ + @Test + public void testFilenameWithOneWord() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke.pdf"; + String searchString = "luke"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains one word + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchOneWord() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "Abamectin"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains last word + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchLastWord() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "prr"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains number + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchNumber() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "1"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains number, blank, underscore and searchString contains whole fileName + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchCompleteFileName() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "1 Abamectin_prr.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + /* + * Filename contains number, blank, underscore and searchString contains part of fileName + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchPartOfFileNameSearchNumber() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "1 Abamectin"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + /* + * Filename contains number, blank, underscore and searchString contains part of fileName + */ + @Test + public void testFilenameWithWhitespaceAndUnderscoreAndSearchLastPartOfFileName() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "1 Abamectin_prr.pdf"; + String searchString = "_prr.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + @Test + public void testFilenameWithNumbersExactMatch() throws IOException { + + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "116 IDD0000261308.pdf"; + String filename2 = "115 IDD0000261308.pdf"; + String searchString = "\"116 IDD\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + indexDocument("template1", "dossierId1", "fileId2", filename2, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + @Test + public void testFilenameWithNumbersMatch() throws IOException { + + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "VV-733382.pdf"; + String searchString = "733382"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + } + + + /* + * Filename contains only one uppercase word and searchString contains complete filename without ending + */ + @Test + public void testFilenameWithOneWordUppercase() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "LUKE.pdf"; + String searchString = "luke"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains only one word and searchString contains complete filename with ending + */ + @Test + public void testFilenameWithOneWordAndEnding() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke.pdf"; + String searchString = "luke.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains only one word with umlaut and searchString contains complete filename without ending + */ + @Test + public void testFilenameWithOneWordWithUmlaut() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "Äpfel.pdf"; + String searchString = "äpfel"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * FFilename contains only one word with umlaut and searchString contains complete filename with ending + */ + @Test + public void testFilenameWithOneWordWithUmlautAndEnding() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "Äpfel.pdf"; + String searchString = "äpfel.pdf"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains one word from filename + */ + @Test + public void testFilenameWithHyphenOneWordSearch() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "luke"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains filename without file ending + */ + @Test + public void testFilenameWithHyphenWithoutFileEnding() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "luke-skywalker-42"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains complete filename within quotes + */ + @Test + public void testFilenameWithHyphenWholeFilenameSearchWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String matchedString = fileName; + String searchString = "\"" + matchedString + "\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains part of filename within quotes + */ + @Test + public void testFilenameWithHyphenOneWordSearchWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String matchedString = "luke"; + String searchString = "\"" + matchedString + "\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Filename contains hyphen and searchString contains filename without file ending within quotes + */ + @Test + public void testFilenameWithHyphenWithoutFileEndingWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String matchedString = "luke-skywalker-42"; + String searchString = "\"luke-skywalker-42\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Filename contains hyphen & blank and searchString contains complete filename within quotes + */ + @Test + public void testFilenameWithHyphenAndBlankWholeFilenameSearchWithQuotes() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-blankwalker 42.pdf"; + String matchedString = fileName; + String searchString = "\"" + matchedString + "\""; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(matchedString)).isTrue(); + } + + + /* + * Search string with hyphen + */ + @Test + public void testSearchStringWithHyphen() throws IOException { + // Arrange + ClassPathResource textResource = new ClassPathResource("files/Text.json"); + Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); + String fileName = "luke-skywalker-42.pdf"; + String searchString = "14C]-SDS-46851"; + indexDocument("template1", "dossierId1", "fileId1", fileName, text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + + // Act + SearchResult result = searchService.search(searchString, null, List.of("dossierId1"), null, null, false, false, null, null, 0, 10, true); + + // Assert + assertThat(result.getMatchedDocuments().size()).isEqualTo(1); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().size()).isGreaterThan(0); + assertThat(result.getMatchedDocuments().get(0).getMatchedTerms().contains(searchString)).isTrue(); + assertThat(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").size()).isGreaterThan(0); + assertThat(StringUtils.contains(result.getMatchedDocuments().get(0).getHighlights().get("sections.text").toArray()[0].toString(), searchString)).isTrue(); + } + + + private void indexDocument(String dossierTemplateId, + String dossierId, + String fileId, + String filename, + Text text, + String assignee, + boolean deleted, + boolean archived, + WorkflowStatus workflowStatus, + Map fileAttributes) { + + var indexDocument = indexDocumentConverterService.convert(dossierTemplateId, + dossierId, + fileId, + filename, + text, + assignee, + deleted, + archived, + workflowStatus, + fileAttributes); + documentIndexService.indexDocument(indexDocument); + } + + + public void updateDocument(String fileId, String assignee, boolean deleted, boolean archived, String workflowStatus, Map fileAttributes) { + + var updateDocument = indexDocumentConverterService.convertUpdateDocument(assignee, deleted, archived, workflowStatus, fileAttributes); + documentUpdateService.updateDocument(fileId, updateDocument); + + } + +}