Marmelator/search-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Pull request #50: RED-3800 - made search service work with apple silicone.

Browse Source 
Merge in RED/search-service from search-index-update-2 to master

* commit 'a8ef4044dc75c4a5e64d9c4e7d53be092ec08e79':
  RED-3800 - fixed tests to work on apple silicone
  RED-3800 - made search service work with apple silicone. Fixed RED-3737
This commit is contained in:
Timo Bejan 2022-04-13 12:22:10 +02:00
commit 9a13d44e8b
11 changed files with 87 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
search-service-image-v1/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<groupId>com.iqser.red</groupId>
<artifactId>platform-docker-dependency</artifactId>
<version>1.1.0</version>
<version>1.2.0</version>
<relativePath />
</parent>
<modelVersion>4.0.0</modelVersion>
@ -42,7 +42,7 @@
<artifactId>docker-maven-plugin</artifactId>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<plugin>
@ -95,4 +95,4 @@
</plugins>
</pluginManagement>
</build>
</project>
</project>

2
search-service-v1/pom.xml
View File

@ -44,7 +44,7 @@
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.9.0.2155</version>
</plugin>
</plugin>
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>

3
search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/MatchedDocument.java
View File

@ -27,6 +27,9 @@ public class MatchedDocument {
private String assignee;
private Map<String, String> fileAttributes;
private String workflowStatus;
private boolean dossierDeleted;
private boolean dossierArchived;
private String fileName;
@Builder.Default
private Map<String, Set<String>> highlights = new HashMap<>();

2
search-service-v1/search-service-server-v1/pom.xml
View File

@ -78,7 +78,7 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>elasticsearch</artifactId>
<version>1.15.3</version>
<version>1.16.3</version>
<scope>test</scope>
</dependency>
<dependency>

8
search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java
View File

@ -62,7 +62,8 @@ public class SearchService {
includeDeletedDossiers, includeArchivedDossiers, workflowStatus, fileAttributes, returnSections))
.from(getPageOrDefault(page) * getPageSizeOrDefault(pageSize))
.size(getPageSizeOrDefault(pageSize))
.fetchSource(new String[]{"dossierId", "dossierTemplateId", "fileId", "assignee", "dossierStatus", "workflowStatus", "fileAttributes"}, new String[]{"sections"})
.fetchSource(new String[]{"dossierId", "dossierTemplateId", "dossierDeleted", "dossierArchived", "filename",
"fileId", "assignee", "dossierStatus", "workflowStatus", "fileAttributes"}, new String[]{"sections"})
.highlighter(new HighlightBuilder().field("sections.text").field("filename").field("fileAttributes.value").highlighterType("fvh"))
.trackScores(true);
@ -233,10 +234,13 @@ public class SearchService {
.assignee((String) hit.getSourceAsMap().get("assignee"))
.fileAttributes(convertFileAttributes(hit.getSourceAsMap().get("fileAttributes")))
.workflowStatus((String) hit.getSourceAsMap().get("workflowStatus"))
.fileName((String) hit.getSourceAsMap().get("fileName"))
.dossierDeleted((Boolean) hit.getSourceAsMap().get("dossierDeleted"))
.dossierArchived((Boolean) hit.getSourceAsMap().get("dossierArchived"))
.highlights(hit.getHighlightFields()
.entrySet()
.stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> Arrays.stream(e.getValue().getFragments())
.collect(Collectors.toMap(Map.Entry::getKey, e -> Arrays.stream(e.getValue().getFragments())
.map(Text::string)
.collect(Collectors.toSet()))))
.matchedTerms(matchesTerms)

85
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java
View File

@ -1,37 +1,61 @@
package com.iqser.red.service.search.v1.server.service;
import java.time.Duration;
import org.assertj.core.util.Lists;
import org.junit.ClassRule;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.annotation.DirtiesContext;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.utility.DockerImageName;
import com.amazonaws.services.s3.AmazonS3;
import com.iqser.red.service.search.v1.server.Application;
import com.iqser.red.service.search.v1.server.client.ElasticsearchClient;
import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import org.junit.ClassRule;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.utility.DockerImageName;
@ComponentScan
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT,
properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS})
@ContextConfiguration(initializers = {AbstractElasticsearchIntegrationTest.Initializer.class})
@EnableFeignClients(basePackageClasses = AbstractElasticsearchIntegrationTest.TestConfiguration.class)
@DirtiesContext
public class AbstractElasticsearchIntegrationTest {
public abstract class AbstractElasticsearchIntegrationTest {
public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=IMMEDIATE";
@ClassRule
public static ElasticsearchContainer elasticsearchContainer = new ElasticsearchContainer(DockerImageName.parse("nexus.iqser.com:5001/bitnami/elasticsearch:7.13.2-debian-10-r1").asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
var esContainer = new ElasticsearchContainer(DockerImageName.parse("elasticsearch:7.17.2").asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
esContainer.start();
String esHost = esContainer.getHttpHostAddress();
// String host = esHost.substring(0, esHost.indexOf(':'));
int port = Integer.parseInt(esHost.substring(esHost.indexOf(':') + 1));
TestPropertyValues.of(
// "elasticsearch.cluster.hosts[0]=" + host,
"elasticsearch.port=" + port
).applyTo(configurableApplicationContext.getEnvironment());
}
}
@Autowired
private IndexCreatorService indexCreationService;
@ -39,27 +63,10 @@ public class AbstractElasticsearchIntegrationTest {
@Autowired
private StorageService storageService;
@MockBean
private AmazonS3 amazonS3;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
@Import(Application.class)
@EnableFeignClients(basePackageClasses = TestConfiguration.class)
@EnableAutoConfiguration(exclude = {StorageAutoConfiguration.class, RabbitAutoConfiguration.class})
public static class TestConfiguration {
@Bean
public ElasticsearchClient elasticsearchClient() {
ElasticsearchSettings elasticsearchSettings = new ElasticsearchSettings();
String host = elasticsearchContainer.getHttpHostAddress();
elasticsearchSettings.setHosts(Lists.newArrayList(host.substring(0, host.indexOf(':'))));
elasticsearchSettings.setPort(Integer.parseInt(host.substring(host.indexOf(':') + 1)));
return new ElasticsearchClient(elasticsearchSettings);
}
@Bean
public IndexCreatorService indexCreationService(ElasticsearchClient elasticsearchClient,
ElasticsearchSettings elasticsearchSettings) {
@ -77,4 +84,4 @@ public class AbstractElasticsearchIntegrationTest {
}
}
}

25
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java
View File

@ -3,9 +3,11 @@ package com.iqser.red.service.search.v1.server.service;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import com.iqser.red.storage.commons.service.StorageService;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource;
@ -14,12 +16,11 @@ import com.iqser.red.storage.commons.service.S3StorageService;
import lombok.SneakyThrows;
public class FileSystemBackedStorageService extends S3StorageService {
public class FileSystemBackedStorageService implements StorageService {
private final Map<String, File> dataMap = new HashMap<>();
public FileSystemBackedStorageService() {
super(null, null);
}
@SneakyThrows
@ -34,6 +35,21 @@ public class FileSystemBackedStorageService extends S3StorageService {
}
@Override
public void deleteObject(String objectId) {
dataMap.remove(objectId);
}
@Override
public boolean objectExists(String objectId) {
return dataMap.containsKey(objectId);
}
@Override
public void init() {
}
@SneakyThrows
@Override
public void storeObject(String objectId, byte[] data) {
@ -44,6 +60,11 @@ public class FileSystemBackedStorageService extends S3StorageService {
dataMap.put(objectId, tempFile);
}
@Override
public void storeObject(String objectId, InputStream stream) {
}
public void clearStorage() {
this.dataMap.forEach((k, v) -> {
v.delete();

2
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/IndexTest.java
View File

@ -17,8 +17,6 @@ import com.iqser.red.service.search.v1.server.queue.IndexingMessageReceiver;
import lombok.SneakyThrows;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS})
public class IndexTest extends AbstractElasticsearchIntegrationTest {
@Autowired

2
search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java
View File

@ -29,8 +29,6 @@ import com.iqser.red.service.search.v1.server.model.Text;
import lombok.SneakyThrows;
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS})
public class SearchTest extends AbstractElasticsearchIntegrationTest {
@Autowired

5
search-service-v1/search-service-server-v1/src/test/resources/application.yml
View File

@ -14,4 +14,7 @@ storage:
bucket-name: 'redaction'
region: 'us-east-1'
endpoint: 'https://s3.amazonaws.com'
backend: 's3'
backend: 's3'
elasticsearch:
hosts:
- 'localhost'

3
search-service-v1/search-service-server-v1/src/test/resources/bootstrap.yml
View File

@ -1,3 +0,0 @@
spring:
application:
name: search-service-v1