Marmelator/search-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

RED-5669: Support both elasticsearch and opensearch

Browse Source
This commit is contained in:
deiflaender 2023-03-03 14:20:55 +01:00
parent b754a1abc5
commit aa22906ade
30 changed files with 1957 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
search-service-v1/search-service-server-v1/pom.xml
View File

@ -51,17 +51,23 @@
<artifactId>jackson-commons</artifactId>
</dependency>
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>8.6.2</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fasterxml.jackson.core</groupId>-->
<!-- <artifactId>jackson-databind</artifactId>-->
<!-- </dependency>-->
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-rest-client</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-java</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>jakarta.json</groupId>
@ -69,10 +75,6 @@
<version>2.0.1</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fasterxml.jackson.dataformat</groupId>-->
<!-- <artifactId>jackson-dataformat-xml</artifactId>-->
<!-- </dependency>-->
<!-- spring -->
<dependency>
@ -107,6 +109,12 @@
<version>1.16.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.opensearch</groupId>
<artifactId>opensearch-testcontainers</artifactId>
<version>2.0.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.amqp</groupId>
<artifactId>spring-rabbit-test</artifactId>

6
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java
View File

@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
import com.iqser.red.service.search.v1.server.client.FileStatusClient;
import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient;
import com.iqser.red.service.search.v1.server.service.opensearch.OpensearchClient;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings;
@ -56,9 +56,9 @@ public class Application {
@Bean
@ConditionalOnMissingBean
public EsClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) {
public OpensearchClient elasticsearchClient(ElasticsearchSettings elasticsearchSettings) {
return new EsClient(elasticsearchSettings);
return new OpensearchClient(elasticsearchSettings);
}

2
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java
View File

@ -6,7 +6,7 @@ import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.search.v1.model.SearchRequest;
import com.iqser.red.service.search.v1.model.SearchResult;
import com.iqser.red.service.search.v1.resources.SearchResource;
import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService;
import com.iqser.red.service.search.v1.server.service.SearchService;
import lombok.RequiredArgsConstructor;

15
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java
View File

@ -24,14 +24,14 @@ import com.iqser.red.service.search.v1.server.client.DossierClient;
import com.iqser.red.service.search.v1.server.client.FileStatusClient;
import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.search.v1.server.model.Text;
import com.iqser.red.service.search.v1.server.service.DocumentDeleteService;
import com.iqser.red.service.search.v1.server.service.DocumentIndexService;
import com.iqser.red.service.search.v1.server.service.DocumentUpdateService;
import com.iqser.red.service.search.v1.server.service.IndexCreatorService;
import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService;
import com.iqser.red.service.search.v1.server.service.IndexInformationService;
import com.iqser.red.service.search.v1.server.service.TextStorageService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -44,14 +44,15 @@ public class IndexingMessageReceiver {
private final ObjectMapper objectMapper;
private final TextStorageService textStorageService;
private final DocumentIndexService documentIndexService;
private final FileStatusClient fileStatusClient;
private final DossierClient dossierClient;
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
private final RabbitTemplate rabbitTemplate;
private final DocumentDeleteService documentDeleteService;
private final DocumentUpdateService documentUpdateService;
private final IndexCreatorService indexCreatorService;
private final RabbitTemplate rabbitTemplate;
private final DocumentIndexService documentIndexService;
private final IndexDeleteService indexDeleteService;
private final IndexInformationService indexInformationService;
private final IndexDocumentConverterService indexDocumentConverterService;

8
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java Normal file
View File

@ -0,0 +1,8 @@
package com.iqser.red.service.search.v1.server.service;
public interface DocumentDeleteService {
void deleteDocument(String fileId);
}

9
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java Normal file
View File

@ -0,0 +1,9 @@
package com.iqser.red.service.search.v1.server.service;
import com.iqser.red.service.search.v1.server.model.IndexDocument;
public interface DocumentIndexService {
void indexDocument(IndexDocument indexDocument);
}

9
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java Normal file
View File

@ -0,0 +1,9 @@
package com.iqser.red.service.search.v1.server.service;
import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate;
public interface DocumentUpdateService {
void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate);
}

7
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java Normal file
View File

@ -0,0 +1,7 @@
package com.iqser.red.service.search.v1.server.service;
public interface IndexCreatorService {
void createIndex();
}

10
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.search.v1.server.service;
public interface IndexDeleteService {
void closeIndex();
void dropIndex();
}

23
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java Normal file
View File

@ -0,0 +1,23 @@
package com.iqser.red.service.search.v1.server.service;
import java.util.List;
import java.util.Map;
import com.iqser.red.service.search.v1.model.SearchResult;
public interface SearchService {
SearchResult search(String queryString,
List<String> dossierTemplateIds,
List<String> dossierIds,
String fileId,
String assignee,
boolean includeDeletedDossiers,
boolean includeArchivedDossiers,
String workflowStatus,
Map<String, String> fileAttributes,
int page,
int pageSize,
boolean returnSections);
}

11
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteService.java → search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentDeleteServiceImpl.java
View File

@ -1,21 +1,25 @@
package com.iqser.red.service.search.v1.server.service.elasticsearch;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME;
import java.io.IOException;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.service.DocumentDeleteService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.Refresh;
import co.elastic.clients.elasticsearch.core.DeleteRequest;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class DocumentDeleteService {
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
public class DocumentDeleteServiceImpl implements DocumentDeleteService {
private final EsClient client;
private final ElasticsearchSettings settings;
@ -23,7 +27,7 @@ public class DocumentDeleteService {
public void deleteDocument(String fileId) {
DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(settings.getRefreshPolicy()).build();
DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build();
try {
client.delete(request);
@ -34,4 +38,3 @@ public class DocumentDeleteService {
}

16
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexService.java → search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentIndexServiceImpl.java
View File

@ -1,20 +1,19 @@
package com.iqser.red.service.search.v1.server.service.elasticsearch;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME;
import java.io.IOException;
import java.util.Map;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.model.IndexDocument;
import com.iqser.red.service.search.v1.server.model.Text;
import com.iqser.red.service.search.v1.server.service.IndexDocumentConverterService;
import com.iqser.red.service.search.v1.server.service.DocumentIndexService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.Refresh;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -22,7 +21,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class DocumentIndexService {
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
public class DocumentIndexServiceImpl implements DocumentIndexService {
private final EsClient client;
private final ElasticsearchSettings settings;
@ -34,11 +34,11 @@ public class DocumentIndexService {
try {
client.index(i -> i.index(INDEX_NAME)
.id(indexDocument.getFileId())
.refresh(settings.getRefreshPolicy())
.refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy()))
.document(indexDocument));
} catch (IOException | ElasticsearchException e) {
throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e);
}
}
}
}

12
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateService.java → search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/DocumentUpdateServiceImpl.java
View File

@ -1,23 +1,27 @@
package com.iqser.red.service.search.v1.server.service.elasticsearch;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME;
import java.io.IOException;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate;
import com.iqser.red.service.search.v1.server.service.DocumentUpdateService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import co.elastic.clients.elasticsearch._types.Refresh;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@Service
@RequiredArgsConstructor
public class DocumentUpdateService {
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
public class DocumentUpdateServiceImpl implements DocumentUpdateService {
private final EsClient client;
private final ElasticsearchSettings settings;
@ -28,10 +32,10 @@ public class DocumentUpdateService {
public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) {
try {
client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(settings.getRefreshPolicy()), IndexDocumentUpdate.class);
client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class);
} catch (IOException | ElasticsearchException e) {
throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e);
}
}
}
}

9
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/EsClient.java
View File

@ -12,11 +12,11 @@ import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
public class EsClient {
// Lower timeouts should be set per request.
@ -35,7 +36,7 @@ public class EsClient {
private final ElasticsearchSettings settings;
@Delegate
private ElasticsearchClient client;
private co.elastic.clients.elasticsearch.ElasticsearchClient client;
@PostConstruct
@ -58,7 +59,7 @@ public class EsClient {
ElasticsearchTransport transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper());
this.client = new ElasticsearchClient(transport);
this.client = new co.elastic.clients.elasticsearch.ElasticsearchClient(transport);
}
@ -69,4 +70,4 @@ public class EsClient {
client.shutdown();
}
}
}

9
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorService.java → search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexCreatorServiceImpl.java
View File

@ -6,9 +6,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.service.IndexCreatorService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import com.iqser.red.service.search.v1.server.utils.ResourceLoader;
@ -20,7 +22,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class IndexCreatorService {
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
public class IndexCreatorServiceImpl implements IndexCreatorService {
public static final String INDEX_NAME = "redaction";
@ -28,7 +31,7 @@ public class IndexCreatorService {
private final ElasticsearchSettings settings;
public IndexCreatorService(EsClient client, ElasticsearchSettings settings) {
public IndexCreatorServiceImpl(EsClient client, ElasticsearchSettings settings) {
this.client = client;
this.settings = settings;
@ -86,4 +89,4 @@ public class IndexCreatorService {
}
}
}
}

9
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteService.java → search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/IndexDeleteServiceImpl.java
View File

@ -1,10 +1,12 @@
package com.iqser.red.service.search.v1.server.service.elasticsearch;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService.INDEX_NAME;
import static com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorServiceImpl.INDEX_NAME;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -13,7 +15,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class IndexDeleteService {
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
public class IndexDeleteServiceImpl implements IndexDeleteService {
private final EsClient client;
@ -43,4 +46,4 @@ public class IndexDeleteService {
}
}
}
}

7
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchService.java → search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/elasticsearch/SearchServiceImpl.java
View File

@ -13,6 +13,7 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -22,6 +23,7 @@ import com.iqser.red.service.search.v1.model.SearchResult;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.model.IndexDocument;
import com.iqser.red.service.search.v1.server.model.Query;
import com.iqser.red.service.search.v1.server.service.SearchService;
import com.iqser.red.service.search.v1.server.utils.QueryStringConverter;
import co.elastic.clients.elasticsearch._types.FieldValue;
@ -43,7 +45,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class SearchService {
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "elasticsearch")
public class SearchServiceImpl implements SearchService {
private final EsClient client;
@ -330,4 +333,4 @@ public class SearchService {
return page;
}
}
}

41
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentDeleteServiceImpl.java Normal file
View File

@ -0,0 +1,41 @@
package com.iqser.red.service.search.v1.server.service.opensearch;
import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME;
import java.io.IOException;
import org.opensearch.client.opensearch._types.OpenSearchException;
import org.opensearch.client.opensearch._types.Refresh;
import org.opensearch.client.opensearch.core.DeleteRequest;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.service.DocumentDeleteService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
public class DocumentDeleteServiceImpl implements DocumentDeleteService {
private final OpensearchClient client;
private final ElasticsearchSettings settings;
public void deleteDocument(String fileId) {
DeleteRequest request = new DeleteRequest.Builder().index(INDEX_NAME).id(fileId).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())).build();
try {
client.delete(request);
} catch (IOException | OpenSearchException e) {
throw new IndexException(String.format(IndexException.DOCUMENT_DELETE_ERROR, fileId), e);
}
}
}

44
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentIndexServiceImpl.java Normal file
View File

@ -0,0 +1,44 @@
package com.iqser.red.service.search.v1.server.service.opensearch;
import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME;
import java.io.IOException;
import org.opensearch.client.opensearch._types.OpenSearchException;
import org.opensearch.client.opensearch._types.Refresh;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.model.IndexDocument;
import com.iqser.red.service.search.v1.server.service.DocumentIndexService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
public class DocumentIndexServiceImpl implements DocumentIndexService {
private final OpensearchClient client;
private final ElasticsearchSettings settings;
@Timed("redactmanager_indexDocument")
public void indexDocument(IndexDocument indexDocument) {
try {
client.index(i -> i.index(INDEX_NAME)
.id(indexDocument.getFileId())
.refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy()))
.document(indexDocument));
} catch (IOException | OpenSearchException e) {
throw new IndexException(String.format(IndexException.DOCUMENT_INDEX_ERROR, indexDocument.getFileId()), e);
}
}
}

41
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/DocumentUpdateServiceImpl.java Normal file
View File

@ -0,0 +1,41 @@
package com.iqser.red.service.search.v1.server.service.opensearch;
import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME;
import java.io.IOException;
import org.opensearch.client.opensearch._types.OpenSearchException;
import org.opensearch.client.opensearch._types.Refresh;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate;
import com.iqser.red.service.search.v1.server.service.DocumentUpdateService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
public class DocumentUpdateServiceImpl implements DocumentUpdateService {
private final OpensearchClient client;
private final ElasticsearchSettings settings;
@SneakyThrows
@Timed("redactmanager_updateDocument")
public void updateDocument(String fileId, IndexDocumentUpdate indexDocumentUpdate) {
try {
client.update(u -> u.index(INDEX_NAME).id(fileId).doc(indexDocumentUpdate).refresh(Refresh._DESERIALIZER.parse(settings.getRefreshPolicy())), IndexDocumentUpdate.class);
} catch (IOException | OpenSearchException e) {
throw new IndexException(String.format(IndexException.DOCUMENT_UPDATE_ERROR, fileId), e);
}
}
}

108
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexCreatorServiceImpl.java Normal file
View File

@ -0,0 +1,108 @@
package com.iqser.red.service.search.v1.server.service.opensearch;
import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import org.opensearch.client.json.JsonpMapper;
import org.opensearch.client.opensearch._types.mapping.TypeMapping;
import org.opensearch.client.opensearch.indices.IndexSettings;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.service.IndexCreatorService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import com.iqser.red.service.search.v1.server.utils.ResourceLoader;
import jakarta.json.stream.JsonParser;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
public class IndexCreatorServiceImpl implements IndexCreatorService {
public static final String INDEX_NAME = "redaction";
private final OpensearchClient client;
private final ElasticsearchSettings settings;
public IndexCreatorServiceImpl(OpensearchClient client, ElasticsearchSettings settings) {
this.client = client;
this.settings = settings;
if (!indexExists()) {
createIndex();
}
}
public void createIndex() {
try {
var response = client.indices().create(i -> i.index(INDEX_NAME).settings(createIndexSettings()).mappings(createIndexMapping()));
log.info("Successfully created index: {}", response.index());
} catch (IOException e) {
log.error("Failed to create index.", e);
}
}
private boolean indexExists() {
try {
var response = client.indices().exists(i -> i.index(INDEX_NAME));
return response.value();
} catch (IOException e) {
throw new IndexException(INDEX_EXISTS_ERROR, e);
}
}
@SneakyThrows
private TypeMapping createIndexMapping() {
URL resource = ResourceLoader.class.getClassLoader().getResource("index/mapping.json");
try (InputStream is = resource.openStream()) {
JsonpMapper mapper = client._transport().jsonpMapper();
JsonParser parser = mapper.jsonProvider().createParser(is);
return TypeMapping._DESERIALIZER.deserialize(parser, mapper);
}
}
@SneakyThrows
private IndexSettings createIndexSettings() {
URL resource = ResourceLoader.class.getClassLoader().getResource("index/settings.json");
try (InputStream is = resource.openStream()) {
JsonpMapper mapper = client._transport().jsonpMapper();
JsonParser parser = mapper.jsonProvider().createParser(is);
var indexSettingsFromJson = IndexSettings._DESERIALIZER.deserialize(parser, mapper);
// It is not possible to set "index.mapping.nested_objects.limit", OpenSearch seems to not have this param.
// Hopefully they don't hava a limit for this, I was not able to find anything.
// As elasticsearch has a limit for this, and we can't set it, it seems this is the only reason for now to have both clients.
var indexSettings = new IndexSettings.Builder().index(indexSettingsFromJson.index())
.numberOfReplicas(settings.getNumberOfReplicas())
.numberOfShards(settings.getNumberOfShards())
.analysis(indexSettingsFromJson.analysis())
.build();
return indexSettings;
}
}
}

49
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/IndexDeleteServiceImpl.java Normal file
View File

@ -0,0 +1,49 @@
package com.iqser.red.service.search.v1.server.service.opensearch;
import static com.iqser.red.service.search.v1.server.service.opensearch.IndexCreatorServiceImpl.INDEX_NAME;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.service.IndexDeleteService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
public class IndexDeleteServiceImpl implements IndexDeleteService {
private final OpensearchClient client;
@SneakyThrows
public void closeIndex() {
var closeIndexResponse = client.indices().close(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m")));
if (closeIndexResponse.acknowledged()) {
log.info("Index is closed");
} else {
throw new IndexException("Error while closing index");
}
}
@SneakyThrows
public void dropIndex() {
log.info("Will drop index");
var deleteIndexResponse = client.indices().delete(i -> i.index(INDEX_NAME).timeout(t -> t.time("2m")));
if (deleteIndexResponse.acknowledged()) {
log.info("Index is dropped");
} else {
throw new IndexException("Error while dropping index");
}
}
}

73
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/OpensearchClient.java Normal file
View File

@ -0,0 +1,73 @@
package com.iqser.red.service.search.v1.server.service.opensearch;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.opensearch.client.RestClient;
import org.opensearch.client.RestClientBuilder;
import org.opensearch.client.json.jackson.JacksonJsonpMapper;
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.transport.rest_client.RestClientTransport;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import lombok.RequiredArgsConstructor;
import lombok.experimental.Delegate;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
public class OpensearchClient {
// Lower timeouts should be set per request.
private static final int ABSURD_HIGH_TIMEOUT = 90_000_000;
private final ElasticsearchSettings settings;
@Delegate
private OpenSearchClient client;
@PostConstruct
public void init() {
HttpHost[] httpHost = settings.getHosts()
.stream()
.map(host -> new HttpHost(host, settings.getPort(), settings.getScheme()))
.collect(Collectors.toList())
.toArray(new HttpHost[settings.getHosts().size()]);
RestClientBuilder builder = RestClient.builder(httpHost)
.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT));
if (settings.getUsername() != null && !settings.getUsername().isEmpty()) {
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword()));
builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
var transport = new RestClientTransport(builder.build(), new JacksonJsonpMapper());
this.client = new OpenSearchClient(transport);
}
@PreDestroy
public void onShutdown() {
client.shutdown();
}
}

338
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/opensearch/SearchServiceImpl.java Normal file
View File

@ -0,0 +1,338 @@
package com.iqser.red.service.search.v1.server.service.opensearch;
import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.opensearch.client.json.JsonData;
import org.opensearch.client.opensearch._types.FieldValue;
import org.opensearch.client.opensearch._types.query_dsl.ChildScoreMode;
import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders;
import org.opensearch.client.opensearch.core.SearchRequest;
import org.opensearch.client.opensearch.core.SearchResponse;
import org.opensearch.client.opensearch.core.search.BuiltinHighlighterType;
import org.opensearch.client.opensearch.core.search.HighlightField;
import org.opensearch.client.opensearch.core.search.HighlighterType;
import org.opensearch.client.opensearch.core.search.Hit;
import org.opensearch.client.opensearch.core.search.InnerHitsResult;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.search.v1.model.MatchedDocument;
import com.iqser.red.service.search.v1.model.MatchedSection;
import com.iqser.red.service.search.v1.model.SearchResult;
import com.iqser.red.service.search.v1.server.exception.IndexException;
import com.iqser.red.service.search.v1.server.model.IndexDocument;
import com.iqser.red.service.search.v1.server.model.Query;
import com.iqser.red.service.search.v1.server.service.SearchService;
import com.iqser.red.service.search.v1.server.utils.QueryStringConverter;
import io.micrometer.core.annotation.Timed;
import io.micrometer.core.instrument.util.StringUtils;
import jakarta.json.JsonObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "search", name = "backend", havingValue = "opensearch")
public class SearchServiceImpl implements SearchService {
private final OpensearchClient client;
@Timed("redactmanager_search")
public SearchResult search(String queryString,
List<String> dossierTemplateIds,
List<String> dossierIds,
String fileId,
String assignee,
boolean includeDeletedDossiers,
boolean includeArchivedDossiers,
String workflowStatus,
Map<String, String> fileAttributes,
int page,
int pageSize,
boolean returnSections) {
Query query = QueryStringConverter.convert(queryString);
Map<String, HighlightField> highlightFieldMap = new HashMap<>();
highlightFieldMap.put("sections.text", new HighlightField.Builder().build());
highlightFieldMap.put("filename", new HighlightField.Builder().build());
highlightFieldMap.put("fileAttributes.value", new HighlightField.Builder().build());
SearchRequest request = new SearchRequest.Builder().query(convertQuery(query,
dossierTemplateIds,
dossierIds,
fileId,
assignee,
includeDeletedDossiers,
includeArchivedDossiers,
workflowStatus,
fileAttributes,
returnSections))
.from(getPageOrDefault(page) * getPageSizeOrDefault(pageSize))
.size(getPageSizeOrDefault(pageSize))
.source(s -> s.filter(f -> f.includes("dossierId",
"dossierTemplateId",
"dossierDeleted",
"dossierArchived",
"filename",
"fileId",
"assignee",
"dossierStatus",
"workflowStatus",
"fileAttributes")))
.highlight(h -> h.type(HighlighterType.of(b -> b.builtin(BuiltinHighlighterType.FastVector))).fields(highlightFieldMap))
.trackScores(true)
.build();
SearchResponse response = execute(request);
return convert(response, query);
}
protected SearchResponse<IndexDocument> execute(SearchRequest searchRequest) {
try {
return client.search(searchRequest, IndexDocument.class);
} catch (IOException e) {
throw new IndexException(FAILED_TO_SEARCH, e);
}
}
private org.opensearch.client.opensearch._types.query_dsl.Query convertQuery(Query query,
List<String> dossierTemplateIds,
List<String> dossierIds,
String fileId,
String assignee,
boolean includeDeletedDossiers,
boolean includeArchivedDossiers,
String workflowStatus,
Map<String, String> fileAttributes,
boolean returnSections) {
var entireQuery = QueryBuilders.bool();
var sectionsQueries = QueryBuilders.bool();
for (String must : query.getMusts()) {
var textPhraseQuery = QueryBuilders.matchPhrase().field("sections.text").query(must.toLowerCase(Locale.ROOT)).queryName(must).build()._toQuery();
var filenamePhraseQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(must.toLowerCase(Locale.ROOT)).queryName("filename." + must).build()._toQuery();
var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value")
.query(must.toLowerCase(Locale.ROOT))
.queryName("fileAttributes." + must).build()._toQuery();
var filenameOrTextMustQuery = QueryBuilders.bool().should(textPhraseQuery).should(filenamePhraseQuery).should(fileAttributesPhraseQuery).build()._toQuery();
entireQuery.must(filenameOrTextMustQuery);
sectionsQueries.should(textPhraseQuery);
}
for (String should : query.getShoulds()) {
var textTermQuery = QueryBuilders.matchPhrase().field("sections.text").query(should.toLowerCase(Locale.ROOT)).queryName(should).build()._toQuery();
var filenameTermQuery = QueryBuilders.matchPhrasePrefix().field("filename").query(should.toLowerCase(Locale.ROOT)).queryName("filename." + should).build()._toQuery();
var fileAttributesPhraseQuery = QueryBuilders.matchPhrase().field("fileAttributes.value")
.query(should.toLowerCase(Locale.ROOT))
.queryName("fileAttributes." + should).build()._toQuery();
entireQuery.should(textTermQuery);
entireQuery.should(filenameTermQuery);
entireQuery.should(fileAttributesPhraseQuery);
sectionsQueries.should(textTermQuery);
}
if (returnSections) {
var nestedQuery = QueryBuilders.nested().scoreMode(ChildScoreMode.Avg)
.queryName("sections")
.query(sectionsQueries.build()._toQuery())
.path("sections")
.innerHits(i -> i.size(100)).build()._toQuery();
entireQuery.should(nestedQuery);
}
var filterQuery = QueryBuilders.bool();
if (dossierTemplateIds != null && !dossierTemplateIds.isEmpty()) {
var dossierTemplateIdQueryBuilder = QueryBuilders.bool();
for (var dossierTemplateId : dossierTemplateIds) {
if (StringUtils.isNotEmpty(dossierTemplateId)) {
dossierTemplateIdQueryBuilder = dossierTemplateIdQueryBuilder.should(QueryBuilders.match().field("dossierTemplateId").query(q -> q.stringValue(dossierTemplateId)).build()._toQuery());
}
}
filterQuery.must(dossierTemplateIdQueryBuilder.build()._toQuery());
}
if (dossierIds != null && !dossierIds.isEmpty()) {
var dossierIdQueryBuilder = QueryBuilders.bool();
for (var dossierId : dossierIds) {
if (StringUtils.isNotEmpty(dossierId)) {
dossierIdQueryBuilder = dossierIdQueryBuilder.should(QueryBuilders.match().field("dossierId").query(q -> q.stringValue(dossierId)).build()._toQuery());
}
}
filterQuery.must(dossierIdQueryBuilder.build()._toQuery());
}
if (StringUtils.isNotEmpty(fileId)) {
filterQuery.must(QueryBuilders.match().field("fileId").query(q -> q.stringValue(fileId)).build()._toQuery());
}
if (StringUtils.isNotEmpty(assignee)) {
filterQuery.must(QueryBuilders.match().field("assignee").query(q -> q.stringValue(assignee)).build()._toQuery());
}
if (includeArchivedDossiers) {
filterQuery.must(QueryBuilders.terms().field("dossierArchived")
.terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery());
} else {
filterQuery.must(QueryBuilders.terms().field("dossierArchived").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery());
}
if (includeDeletedDossiers) {
filterQuery.must(QueryBuilders.terms().field("dossierDeleted")
.terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(true).build(), new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery());
} else {
filterQuery.must(QueryBuilders.terms().field("dossierDeleted").terms(t -> t.value(List.of(new FieldValue.Builder().booleanValue(false).build()))).build()._toQuery());
}
if (StringUtils.isNotEmpty(workflowStatus)) {
filterQuery.must(QueryBuilders.match().field("workflowStatus").query(q -> q.stringValue(workflowStatus)).build()._toQuery());
}
if (fileAttributes != null && !fileAttributes.isEmpty()) {
var fileAttributesQueryBuilder = QueryBuilders.bool();
for (var fileAttributeKey : fileAttributes.keySet()) {
if (StringUtils.isNotEmpty(fileAttributeKey)) {
fileAttributesQueryBuilder.filter(List.of(QueryBuilders.bool()
.must(QueryBuilders.match().field("fileAttributes.name").query(q -> q.stringValue(fileAttributeKey)).build()._toQuery())
.must(QueryBuilders.match().field("fileAttributes.value").query(q -> q.stringValue(fileAttributes.get(fileAttributeKey))).build()._toQuery())
.build()
._toQuery()));
}
}
filterQuery.must(fileAttributesQueryBuilder.build()._toQuery());
}
return QueryBuilders.bool().filter(filterQuery.build()._toQuery()).must(entireQuery.build()._toQuery()).build()._toQuery();
}
private SearchResult convert(SearchResponse response, Query query) {
List<Hit> hits = response.hits().hits();
return SearchResult.builder()
.matchedDocuments(hits.stream().map(hit -> convertSearchHit((Hit) hit, query)).collect(Collectors.toList()))
.maxScore(response.maxScore() == null ? 0 :response.maxScore().floatValue())
.total(response.hits().total().value())
.build();
}
private MatchedDocument convertSearchHit(Hit hit, Query query) {
List<String> m = hit.matchedQueries();
Set<String> matchesTerms = m.stream()
.map(match -> match.contains("filename.") ? match.replace("filename.", "") : match)
.map(match -> match.contains("fileAttributes.") ? match.replace("fileAttributes.", "") : match)
.collect(Collectors.toSet());
Set<String> unmatchedTerms = Stream.concat(query.getMusts().stream(), query.getShoulds().stream()).filter(term -> !matchesTerms.contains(term)).collect(Collectors.toSet());
IndexDocument indexDocument = (IndexDocument) hit.source();
MatchedDocument.MatchedDocumentBuilder matchedDocumentBuilder = MatchedDocument.builder()
.score(hit.score().floatValue())
.dossierId(indexDocument.getDossierId())
.dossierTemplateId(indexDocument.getDossierTemplateId())
.fileId(indexDocument.getFileId())
.assignee(indexDocument.getAssignee())
.fileAttributes(convertFileAttributes(indexDocument.getFileAttributes()))
.workflowStatus(indexDocument.getWorkflowStatus())
.fileName(indexDocument.getFilename())
.dossierDeleted(indexDocument.isDossierDeleted())
.dossierArchived(indexDocument.isDossierArchived())
.highlights(hit.highlight())
.matchedTerms(matchesTerms)
.unmatchedTerms(unmatchedTerms);
if (hit.innerHits() != null && !hit.innerHits().isEmpty()) {
InnerHitsResult sectionHits = (InnerHitsResult) hit.innerHits().get("sections");
matchedDocumentBuilder.matchedSections(sectionHits.hits().hits().stream().map(innerHit -> convertInnerHit(innerHit)).collect(Collectors.toList()))
.containsAllMatchedSections(sectionHits.hits().total().value() == sectionHits.hits().hits().size());
}
return matchedDocumentBuilder.build();
}
private Map<String, String> convertFileAttributes(Object fileAttributesSourceMap) {
Map<String, String> fileAttributes = new HashMap<>();
if (fileAttributesSourceMap != null) {
List<HashMap<String, String>> list = new ObjectMapper().convertValue(fileAttributesSourceMap, ArrayList.class);
list.forEach(r -> fileAttributes.put(r.get("name"), r.get("value")));
}
return fileAttributes;
}
private MatchedSection convertInnerHit(Hit<JsonData> hit) {
JsonObject indexSection = hit.source().toJson().asJsonObject();
var jsonArray = indexSection.getJsonArray("pages");
var pages = IntStream.range(0, jsonArray.size()).mapToObj(i -> jsonArray.getInt(i)).collect(Collectors.toSet());
return MatchedSection.builder()
.headline(indexSection.getString("headline"))
.sectionNumber(indexSection.getInt("sectionNumber"))
.pages(pages)
.matchedTerms(hit.matchedQueries().stream().collect(Collectors.toSet()))
.build();
}
private int getPageSizeOrDefault(int pageSize) {
if (pageSize <= 0) {
return 10;
}
return pageSize;
}
private int getPageOrDefault(int page) {
if (page < 0) {
return 0;
}
return page;
}
}

3
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java
View File

@ -7,7 +7,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import co.elastic.clients.elasticsearch._types.Refresh;
import lombok.Data;
@Data
@ -34,6 +33,6 @@ public class ElasticsearchSettings {
/**
* ES refresh policy for write requests to use. Used in tests to wait for completion of write requests.
*/
private Refresh refreshPolicy = Refresh.True;
private String refreshPolicy = "true";
}

3
search-service-v1/search-service-server-v1/src/main/resources/application.yml
View File

@ -34,6 +34,9 @@ management:
endpoints.web.exposure.include: prometheus, health
metrics.export.prometheus.enabled: ${monitoring.enabled:false}
search:
backend: elasticsearch
elasticsearch:
hosts:
- ${elasticsearch.cluster.hosts}

32
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java
View File

@ -1,6 +1,6 @@
package com.iqser.red.service.search.v1.server.service;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@ -15,26 +15,24 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.context.junit4.SpringRunner;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.utility.DockerImageName;
import com.iqser.red.service.search.v1.server.Application;
import com.iqser.red.service.search.v1.server.service.elasticsearch.EsClient;
import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexCreatorService;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
@ComponentScan
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS})
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractElasticsearchIntegrationTest.SEARCH_BACKEND})
@ContextConfiguration(initializers = {AbstractElasticsearchIntegrationTest.Initializer.class})
@EnableFeignClients(basePackageClasses = AbstractElasticsearchIntegrationTest.TestConfiguration.class)
@DirtiesContext
public abstract class AbstractElasticsearchIntegrationTest {
public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=WaitFor";
public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for";
public static final String SEARCH_BACKEND = "search.backend=elasticsearch";
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
@ -45,19 +43,13 @@ public abstract class AbstractElasticsearchIntegrationTest {
esContainer.start();
String esHost = esContainer.getHttpHostAddress();
// String host = esHost.substring(0, esHost.indexOf(':'));
int port = Integer.parseInt(esHost.substring(esHost.indexOf(':') + 1));
int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1));
TestPropertyValues.of(
// "elasticsearch.cluster.hosts[0]=" + host,
"elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment());
TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment());
}
}
@Autowired
protected IndexCreatorService indexCreationService;
@Autowired
protected StorageService storageService;
@ -65,14 +57,6 @@ public abstract class AbstractElasticsearchIntegrationTest {
@EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
public static class TestConfiguration {
@Bean
public IndexCreatorService indexCreationService(EsClient elasticsearchClient, ElasticsearchSettings elasticsearchSettings) {
return new IndexCreatorService(elasticsearchClient, elasticsearchSettings);
}
@Bean
@Primary
public StorageService inmemoryStorage() {

69
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractOpensearchIntegrationTest.java Normal file
View File

@ -0,0 +1,69 @@
package com.iqser.red.service.search.v1.server.service;
import org.junit.runner.RunWith;
import org.opensearch.testcontainers.OpensearchContainer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import org.testcontainers.utility.DockerImageName;
import com.iqser.red.service.search.v1.server.Application;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
@ComponentScan
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractOpensearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS, AbstractOpensearchIntegrationTest.SEARCH_BACKEND})
@ContextConfiguration(initializers = {AbstractOpensearchIntegrationTest.Initializer.class})
@EnableFeignClients(basePackageClasses = AbstractOpensearchIntegrationTest.TestConfiguration.class)
@DirtiesContext
public abstract class AbstractOpensearchIntegrationTest {
public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=wait_for";
public static final String SEARCH_BACKEND = "search.backend=opensearch";
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
var esContainer = new OpensearchContainer(DockerImageName.parse("opensearchproject/opensearch:2.6.0"));
esContainer.start();
String esHost = esContainer.getHttpHostAddress();
int port = Integer.parseInt(esHost.substring(esHost.lastIndexOf(':') + 1));
TestPropertyValues.of("elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment());
}
}
@Autowired
protected StorageService storageService;
@Configuration
@EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
public static class TestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
}
}
}

7
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java → search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/ElasticsearchTest.java
View File

@ -22,15 +22,10 @@ import com.iqser.red.service.search.v1.server.client.FileStatusClient;
import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.search.v1.server.client.IndexInformationClient;
import com.iqser.red.service.search.v1.server.model.Text;
import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentDeleteService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentIndexService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.DocumentUpdateService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.IndexDeleteService;
import com.iqser.red.service.search.v1.server.service.elasticsearch.SearchService;
import lombok.SneakyThrows;
public class SearchTest extends AbstractElasticsearchIntegrationTest {
public class ElasticsearchTest extends AbstractElasticsearchIntegrationTest {
@Autowired
private ObjectMapper objectMapper;

1041
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/OpensearchTest.java Normal file
View File

File diff suppressed because it is too large Load Diff