RED-809: Implemented workflow logic

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -105,6 +105,7 @@ public class PlanSpec {
                                 .dockerConfiguration(
                                         new DockerConfiguration()
                                                 .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                                .dockerRunArguments("--net=host")
                                                 .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
                                                 .volume("/var/run/docker.sock", "/var/run/docker.sock")
                                 )

search-service-image-v1/src/main/docker/Dockerfile

@@ -4,15 +4,4 @@ ARG PLATFORM_JAR
 
 ENV PLATFORM_JAR ${PLATFORM_JAR}
 
-ENV USES_ELASTICSEARCH false
-
 COPY ["${PLATFORM_JAR}", "/"]
-
-RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    wget cabextract xfonts-utils fonts-liberation \
- && rm -rf /var/lib/apt/lists/*
-
-RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.7_all.deb -o /tmp/ttf-mscorefonts-installer_3.7_all.deb \
- && dpkg -i /tmp/ttf-mscorefonts-installer_3.7_all.deb \
- && rm /tmp/ttf-mscorefonts-installer_3.7_all.deb \

search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/IndexMessage.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.search.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class IndexMessage {
+
+    private String dossierId;
+    private String fileId;
+
+}

search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchRequest.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.search.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SearchRequest {
+
+    private String queryString;
+    private String dossierId;
+    private String fileId;
+    private Integer from;
+    private Integer size;
+    private boolean returnSections;
+
+}

search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/resources/SearchResource.java

@@ -0,0 +1,22 @@
+package com.iqser.red.service.search.v1.resources;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+import com.iqser.red.service.search.v1.model.SearchRequest;
+import com.iqser.red.service.search.v1.model.SearchResult;
+
+public interface SearchResource {
+
+    String SEARCH_PATH = "/search";
+
+    @ResponseBody
+    @ResponseStatus(value = HttpStatus.OK)
+    @GetMapping(value = SEARCH_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
+    SearchResult getDossierStatus(@RequestBody SearchRequest searchRequest);
+
+}

search-service-v1/search-service-server-v1/pom.xml

@@ -21,6 +21,17 @@
             <artifactId>search-service-api-v1</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.iqser.red.service</groupId>
+            <artifactId>file-management-service-api-v1</artifactId>
+            <version>2.47.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.iqser.red.service</groupId>
+                    <artifactId>configuration-service-api-v1</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
 
         <!-- commons -->
         <dependency>

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java

@@ -5,15 +5,20 @@ import org.springframework.boot.actuate.autoconfigure.security.servlet.Managemen
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
 
 import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.search.v1.server.client.ElasticsearchClient;
+import com.iqser.red.service.search.v1.server.client.FileStatusClient;
 import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings;
 import com.iqser.red.service.search.v1.server.utils.FailfastElasticSearchHealthIndicator;
 
 @Import({DefaultWebMvcConfiguration.class})
+@EnableFeignClients(basePackageClasses = FileStatusClient.class)
+@EnableConfigurationProperties(ElasticsearchSettings.class)
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
 public class Application {
 

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.search.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.file.management.v1.api.resources.StatusResource;
+
+@FeignClient(name = "StatusResource", url = "${file-management-service.url}")
+public interface FileStatusClient extends StatusResource {
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusProcessingUpdateClient.java

@@ -0,0 +1,9 @@
+package com.iqser.red.service.search.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.file.management.v1.api.resources.FileStatusProcessingUpdateResource;
+
+@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${file-management-service.url}")
+public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java

@@ -0,0 +1,26 @@
+package com.iqser.red.service.search.v1.server.controller;
+
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.iqser.red.service.search.v1.model.SearchRequest;
+import com.iqser.red.service.search.v1.model.SearchResult;
+import com.iqser.red.service.search.v1.resources.SearchResource;
+import com.iqser.red.service.search.v1.server.service.SearchService;
+
+import lombok.RequiredArgsConstructor;
+
+@RestController
+@RequiredArgsConstructor
+public class SearchController implements SearchResource {
+
+    private final SearchService searchService;
+
+
+    public SearchResult getDossierStatus(@RequestBody SearchRequest searchRequest) {
+
+        return searchService.search(searchRequest.getQueryString(), searchRequest.getDossierId(), searchRequest.getFileId(), searchRequest
+                .getFrom(), searchRequest.getSize(), searchRequest.isReturnSections());
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java

@@ -0,0 +1,60 @@
+package com.iqser.red.service.search.v1.server.queue;
+
+import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_DQL;
+import static com.iqser.red.service.search.v1.server.queue.MessagingConfiguration.INDEXING_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.file.management.v1.api.model.FileStatus;
+import com.iqser.red.service.search.v1.model.IndexMessage;
+import com.iqser.red.service.search.v1.server.client.FileStatusClient;
+import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.search.v1.server.model.Text;
+import com.iqser.red.service.search.v1.server.service.DocumentIndexService;
+import com.iqser.red.service.search.v1.server.service.TextStorageService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class IndexingMessageReceiver {
+
+    private final ObjectMapper objectMapper;
+    private final TextStorageService textStorageService;
+    private final DocumentIndexService documentIndexService;
+    private final FileStatusClient fileStatusClient;
+    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
+
+
+    @RabbitHandler
+    @RabbitListener(queues = INDEXING_QUEUE)
+    public void receiveIndexingRequest(String in) throws JsonProcessingException {
+
+        var indexRequest = objectMapper.readValue(in, IndexMessage.class);
+        log.info("Processing indexing request: {}", indexRequest);
+        fileStatusProcessingUpdateClient.indexing(indexRequest.getDossierId(), indexRequest.getFileId());
+        FileStatus fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId());
+        Text text = textStorageService.getText(indexRequest.getDossierId(), indexRequest.getDossierId());
+        documentIndexService.indexDocument(indexRequest.getDossierId(), indexRequest.getFileId(), fileStatus.getFilename(), text);
+        fileStatusProcessingUpdateClient.indexingSuccessful(indexRequest.getDossierId(), indexRequest.getFileId());
+        log.info("Successfully indexed {}", indexRequest);
+
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = INDEXING_DQL)
+    public void receiveIndexingRequestDQL(String in) throws JsonProcessingException {
+
+        var indexRequest = objectMapper.readValue(in, IndexMessage.class);
+        fileStatusProcessingUpdateClient.indexingFailed(indexRequest.getDossierId(), indexRequest.getFileId());
+        log.info("Failed to process indexing request: {}", indexRequest);
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java

@@ -0,0 +1,35 @@
+package com.iqser.red.service.search.v1.server.queue;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import lombok.RequiredArgsConstructor;
+
+@Configuration
+@RequiredArgsConstructor
+public class MessagingConfiguration {
+
+    public static final String INDEXING_QUEUE = "indexingQueue";
+    public static final String INDEXING_DQL = "indexingDQL";
+
+
+    @Bean
+    public Queue indexingQueue() {
+
+        return QueueBuilder.durable(INDEXING_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", INDEXING_DQL)
+                .maxPriority(2)
+                .build();
+    }
+
+
+    @Bean
+    public Queue indexingDeadLetterQueue() {
+
+        return QueueBuilder.durable(INDEXING_DQL).build();
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java

@@ -1,14 +1,11 @@
 package com.iqser.red.service.search.v1.server.service;
 
 import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME;
-import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
-import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 
 import java.io.IOException;
-import java.util.Optional;
+import java.time.OffsetDateTime;
 import java.util.stream.Collectors;
 
-import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -69,6 +66,7 @@ public class DocumentIndexService {
                 .fileId(fileId)
                 .filename(filename)
                 .sections(text.getSectionTexts().stream().map(this::convert).collect(Collectors.toList()))
+                .date(OffsetDateTime.now())
                 .build();
     }
 

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java

@@ -0,0 +1,52 @@
+package com.iqser.red.service.search.v1.server.service;
+
+import java.io.IOException;
+
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.file.management.v1.api.model.FileType;
+import com.iqser.red.service.search.v1.server.model.Text;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class TextStorageService {
+
+    private final StorageService storageService;
+    private final ObjectMapper objectMapper;
+
+
+    public Text getText(String dossierId, String fileId) {
+
+        InputStreamResource inputStreamResource;
+        try {
+            inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
+        } catch (StorageObjectDoesNotExist e) {
+            throw new RuntimeException("Text is not available", e);
+        }
+
+        try {
+            return objectMapper.readValue(inputStreamResource.getInputStream(), Text.class);
+        } catch (IOException e) {
+            throw new RuntimeException("Could not convert Text", e);
+        }
+    }
+
+
+    public static class StorageIdUtils {
+
+        public static String getStorageId(String dossierId, String fileId, FileType fileType) {
+
+            return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
+        }
+
+    }
+
+}

search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java

@@ -6,10 +6,12 @@ import java.util.List;
 import org.elasticsearch.action.support.WriteRequest;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
 
 import lombok.Data;
 
 @Data
+@Primary
 @Configuration
 @ConfigurationProperties("elasticsearch")
 public class ElasticsearchSettings {

search-service-v1/search-service-server-v1/src/main/resources/application.yml

@@ -1,6 +1,8 @@
 info:
   description: Search Service Server V1
 
+file-management-service.url: "http://file-management-service-v1:8080"
+
 server:
   port: 8080
 
@@ -30,6 +32,13 @@ management:
   endpoints.web.exposure.include: prometheus, health
   metrics.export.prometheus.enabled: ${monitoring.enabled:false}
 
+elasticsearch:
+  hosts:
+    - ${elasticsearch.cluster.hosts}
+  port: ${elasticsearch.cluster.port}
+  scheme: ${elasticsearch.cluster.scheme}
+  username: ${elasticsearch.cluster.username}
+  password: ${elasticsearch.cluster.password}
 
 storage:
   signer-type: 'AWSS3V4SignerType'

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java

@@ -1,9 +1,12 @@
 package com.iqser.red.service.search.v1.server.service;
 
+import java.time.Duration;
+
 import org.assertj.core.util.Lists;
 import org.junit.ClassRule;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
@@ -25,6 +28,7 @@ public class AbstractElasticsearchIntegrationTest {
 
     public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=IMMEDIATE";
 
+
     @ClassRule
     public static ElasticsearchContainer elasticsearchContainer = new ElasticsearchContainer(DockerImageName.parse("nexus.iqser.com:5001/bitnami/elasticsearch:7.13.2-debian-10-r1").asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"));
 
@@ -38,7 +42,7 @@ public class AbstractElasticsearchIntegrationTest {
     private AmazonS3 amazonS3;
 
     @Configuration
-    @EnableAutoConfiguration()
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
     @Import(Application.class)
     @EnableFeignClients(basePackageClasses = TestConfiguration.class)
     public static class TestConfiguration {

search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/IndexCreatorTest.java

@@ -1,16 +1,20 @@
 package com.iqser.red.service.search.v1.server.service;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import java.io.IOException;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.search.v1.model.SearchResult;
+import com.iqser.red.service.search.v1.server.client.FileStatusClient;
+import com.iqser.red.service.search.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.search.v1.server.model.Text;
 
 @RunWith(SpringRunner.class)
@@ -26,6 +30,12 @@ public class IndexCreatorTest extends AbstractElasticsearchIntegrationTest {
     @Autowired
     private SearchService searchService;
 
+    @MockBean
+    private FileStatusClient fileStatusClient;
+
+    @MockBean
+    private FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
+
 
     @Test
     public void test() throws IOException, InterruptedException {
@@ -39,12 +49,9 @@ public class IndexCreatorTest extends AbstractElasticsearchIntegrationTest {
         documentIndexService.indexDocument("dossierId", "fileId", "Single Study - Oral (Gavage) Mouse.pdf", text);
         documentIndexService.indexDocument("dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", text2);
 
-        Thread.sleep(1000);
-
-        SearchResult result = searchService.search("hans klaus single" ,null, null, 0 , 10, false);
-
-        System.out.println(result);
+        SearchResult result = searchService.search("hans klaus single" ,null, null, 0 , 10, true);
 
+        assertThat(result.getMatchedDocuments().size()).isEqualTo(2);
     }
 
 }