diff --git a/bamboo-specs/pom.xml b/bamboo-specs/pom.xml index 42e07d8..d35f676 100644 --- a/bamboo-specs/pom.xml +++ b/bamboo-specs/pom.xml @@ -1,37 +1,37 @@ - 4.0.0 + 4.0.0 - - com.atlassian.bamboo - bamboo-specs-parent - 8.1.3 - - + + com.atlassian.bamboo + bamboo-specs-parent + 8.1.3 + + - bamboo-specs - 1.0.0-SNAPSHOT - jar + bamboo-specs + 1.0.0-SNAPSHOT + jar - - - com.atlassian.bamboo - bamboo-specs-api - - - com.atlassian.bamboo - bamboo-specs - + + + com.atlassian.bamboo + bamboo-specs-api + + + com.atlassian.bamboo + bamboo-specs + - - - junit - junit - test - - + + + junit + junit + test + + - - + + diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java index 3a1e2a4..e24d36a 100644 --- a/bamboo-specs/src/main/java/buildjob/PlanSpec.java +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -1,9 +1,12 @@ package buildjob; +import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask; + import java.time.LocalTime; import com.atlassian.bamboo.specs.api.BambooSpec; import com.atlassian.bamboo.specs.api.builders.BambooKey; +import com.atlassian.bamboo.specs.api.builders.Variable; import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration; import com.atlassian.bamboo.specs.api.builders.permission.PermissionType; import com.atlassian.bamboo.specs.api.builders.permission.Permissions; @@ -16,18 +19,15 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup; import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement; import com.atlassian.bamboo.specs.api.builders.project.Project; import com.atlassian.bamboo.specs.builders.task.CheckoutItem; -import com.atlassian.bamboo.specs.api.builders.Variable; import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask; +import com.atlassian.bamboo.specs.builders.task.ScriptTask; import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask; import com.atlassian.bamboo.specs.builders.task.VcsTagTask; import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger; import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger; import com.atlassian.bamboo.specs.model.task.InjectVariablesScope; -import com.atlassian.bamboo.specs.util.BambooServer; -import com.atlassian.bamboo.specs.builders.task.ScriptTask; import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location; - -import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask; +import com.atlassian.bamboo.specs.util.BambooServer; /** * Plan configuration for Bamboo. @@ -38,10 +38,11 @@ public class PlanSpec { private static final String SERVICE_NAME = "search-service"; - private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 "; + private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 "; private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", ""); + /** * Run main to publish plan on Bamboo */ @@ -60,9 +61,15 @@ public class PlanSpec { bambooServer.publish(secPlanPermission); } + private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) { - Permissions permission = new Permissions() - .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD) + + Permissions permission = new Permissions().userPermissions("atlbamboo", + PermissionType.EDIT, + PermissionType.VIEW, + PermissionType.ADMIN, + PermissionType.CLONE, + PermissionType.BUILD) .groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) .groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) .loggedInUserPermissions(PermissionType.VIEW) @@ -70,106 +77,53 @@ public class PlanSpec { return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission); } + private Project project() { - return new Project() - .name("RED") - .key(new BambooKey("RED")); + + return new Project().name("RED").key(new BambooKey("RED")); } + public Plan createPlan() { - return new Plan( - project(), - SERVICE_NAME, new BambooKey(SERVICE_KEY)) - .description("Plan created from (enter repository url of your plan)") + + return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)") .variables(new Variable("maven_add_param", "")) - .stages(new Stage("Default Stage") - .jobs(new Job("Default Job", - new BambooKey("JOB1")) - .tasks( - new ScriptTask() - .description("Clean") - .inlineBody("#!/bin/bash\n" + - "set -e\n" + - "rm -rf ./*"), - new VcsCheckoutTask() - .description("Checkout Default Repository") - .cleanCheckout(true) - .checkoutItems(new CheckoutItem().defaultRepository()), - new ScriptTask() - .description("Build") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh") - .argument(SERVICE_NAME), - createJUnitParserTask() - .description("Resultparser") - .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml") - .enabled(true), - new InjectVariablesTask() - .description("Inject git Tag") - .path("git.tag") - .namespace("g") - .scope(InjectVariablesScope.LOCAL), - new VcsTagTask() - .description("${bamboo.g.gitTag}") - .tagName("${bamboo.g.gitTag}") - .defaultRepository()) - .dockerConfiguration( - new DockerConfiguration() - .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") - .dockerRunArguments("--net=host") - .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml") - .volume("/var/run/docker.sock", "/var/run/docker.sock") - ) - ) - ) + .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean") + .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), + new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask().description("Build").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh").argument(SERVICE_NAME), + createJUnitParserTask().description("Resultparser") + .resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml") + .enabled(true), + new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL), + new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository()) + .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") + .dockerRunArguments("--net=host") + .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) .linkedRepositories("RED / " + SERVICE_NAME) .triggers(new BitbucketServerTrigger()) - .planBranchManagement(new PlanBranchManagement() - .createForVcsBranch() - .delete(new BranchCleanup() - .whenInactiveInRepositoryAfterDays(14)) + .planBranchManagement(new PlanBranchManagement().createForVcsBranch() + .delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)) .notificationForCommitters()); } + public Plan createSecBuild() { - return new Plan( - project(), - SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")) - .description("Security Analysis Plan") - .stages(new Stage("Default Stage") - .jobs(new Job("Default Job", - new BambooKey("JOB1")) - .tasks( - new ScriptTask() - .description("Clean") - .inlineBody("#!/bin/bash\n" + - "set -e\n" + - "rm -rf ./*"), - new VcsCheckoutTask() - .description("Checkout Default Repository") - .cleanCheckout(true) - .checkoutItems(new CheckoutItem().defaultRepository()), - new ScriptTask() - .description("Sonar") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh") - .argument(SERVICE_NAME)) - .dockerConfiguration( - new DockerConfiguration() - .image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") - .dockerRunArguments("--net=host") - .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml") - .volume("/var/run/docker.sock", "/var/run/docker.sock") - ) - ) - ) + + return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan") + .stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean") + .inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), + new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME)) + .dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim") + .dockerRunArguments("--net=host") + .volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) .linkedRepositories("RED / " + SERVICE_NAME) - .triggers( - new ScheduledTrigger() - .scheduleOnceDaily(LocalTime.of(23, 00))) - .planBranchManagement(new PlanBranchManagement() - .createForVcsBranchMatching("release.*") - .notificationForCommitters()); + .triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00))) + .planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters()); } + } diff --git a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java index ed095c7..12d4aec 100644 --- a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java +++ b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java @@ -1,6 +1,5 @@ package buildjob; - import org.junit.Test; import com.atlassian.bamboo.specs.api.builders.plan.Plan; @@ -8,12 +7,15 @@ import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException; import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders; public class PlanSpecTest { + @Test public void checkYourPlanOffline() throws PropertiesValidationException { + Plan plan = new PlanSpec().createPlan(); EntityPropertiesBuilders.build(plan); Plan secPlan = new PlanSpec().createSecBuild(); EntityPropertiesBuilders.build(secPlan); } + } \ No newline at end of file diff --git a/search-service-image-v1/pom.xml b/search-service-image-v1/pom.xml index a06d541..c043916 100644 --- a/search-service-image-v1/pom.xml +++ b/search-service-image-v1/pom.xml @@ -6,7 +6,7 @@ com.iqser.red platform-docker-dependency 1.2.0 - + 4.0.0 diff --git a/search-service-v1/pom.xml b/search-service-v1/pom.xml index 3512ffc..14bde6b 100644 --- a/search-service-v1/pom.xml +++ b/search-service-v1/pom.xml @@ -6,7 +6,7 @@ platform-dependency com.iqser.red 1.10.0 - + 4.0.0 @@ -44,7 +44,7 @@ org.sonarsource.scanner.maven sonar-maven-plugin 3.9.0.2155 - + org.owasp dependency-check-maven diff --git a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchRequest.java b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchRequest.java index ad6b01d..598ed27 100644 --- a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchRequest.java +++ b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchRequest.java @@ -1,13 +1,13 @@ package com.iqser.red.service.search.v1.model; +import java.util.List; +import java.util.Map; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; -import java.util.Map; - @Data @Builder @NoArgsConstructor diff --git a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchResult.java b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchResult.java index d502b89..2ac4011 100644 --- a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchResult.java +++ b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/model/SearchResult.java @@ -21,5 +21,4 @@ public class SearchResult { @Builder.Default private List matchedDocuments = new ArrayList<>(); - } diff --git a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/resources/SearchResource.java b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/resources/SearchResource.java index 27bb877..b63c129 100644 --- a/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/resources/SearchResource.java +++ b/search-service-v1/search-service-api-v1/src/main/java/com/iqser/red/service/search/v1/resources/SearchResource.java @@ -14,6 +14,7 @@ public interface SearchResource { String SEARCH_PATH = "/search"; + @ResponseBody @ResponseStatus(value = HttpStatus.OK) @GetMapping(value = SEARCH_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java index fff4295..80e76dd 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/Application.java @@ -1,10 +1,5 @@ package com.iqser.red.service.search.v1.server; -import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.client.FileStatusClient; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -15,6 +10,12 @@ import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import com.iqser.red.commons.spring.DefaultWebMvcConfiguration; +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.client.FileStatusClient; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.search.v1.server.settings.SearchServiceSettings; + import io.micrometer.core.aop.TimedAspect; import io.micrometer.core.instrument.MeterRegistry; @@ -40,8 +41,8 @@ public class Application { @Bean public TimedAspect timedAspect(MeterRegistry registry) { + return new TimedAspect(registry); } - } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/DossierClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/DossierClient.java index acf1be9..85bbce0 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/DossierClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/DossierClient.java @@ -1,8 +1,9 @@ package com.iqser.red.service.search.v1.server.client; -import com.iqser.red.service.persistence.service.v1.api.resources.DossierResource; import org.springframework.cloud.openfeign.FeignClient; +import com.iqser.red.service.persistence.service.v1.api.resources.DossierResource; + @FeignClient(name = "DossierResource", url = "${persistence-service.url}") public interface DossierClient extends DossierResource { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java index c48221e..effab4e 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/ElasticsearchClient.java @@ -1,9 +1,11 @@ package com.iqser.red.service.search.v1.server.client; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import lombok.RequiredArgsConstructor; -import lombok.experimental.Delegate; -import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; @@ -14,10 +16,11 @@ import org.opensearch.client.RestClientBuilder; import org.opensearch.client.RestHighLevelClient; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.stream.Collectors; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @@ -43,13 +46,11 @@ public class ElasticsearchClient { .toArray(new HttpHost[settings.getHosts().size()]); RestClientBuilder builder = RestClient.builder(httpHost) - .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT) - .setSocketTimeout(ABSURD_HIGH_TIMEOUT)); + .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setConnectTimeout(ABSURD_HIGH_TIMEOUT).setSocketTimeout(ABSURD_HIGH_TIMEOUT)); if (settings.getUsername() != null && !settings.getUsername().isEmpty()) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings - .getPassword())); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(settings.getUsername(), settings.getPassword())); builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusClient.java index a33b1a3..4c800ac 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusClient.java @@ -1,8 +1,9 @@ package com.iqser.red.service.search.v1.server.client; -import com.iqser.red.service.persistence.service.v1.api.resources.StatusResource; import org.springframework.cloud.openfeign.FeignClient; +import com.iqser.red.service.persistence.service.v1.api.resources.StatusResource; + @FeignClient(name = "StatusResource", url = "${persistence-service.url}") public interface FileStatusClient extends StatusResource { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusProcessingUpdateClient.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusProcessingUpdateClient.java index 4d6a462..adbe3f0 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusProcessingUpdateClient.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/client/FileStatusProcessingUpdateClient.java @@ -1,8 +1,10 @@ package com.iqser.red.service.search.v1.server.client; -import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource; import org.springframework.cloud.openfeign.FeignClient; +import com.iqser.red.service.persistence.service.v1.api.resources.FileStatusProcessingUpdateResource; + @FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}") public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource { + } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java index ffd27bb..c5949b7 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/controller/SearchController.java @@ -1,13 +1,14 @@ package com.iqser.red.service.search.v1.server.controller; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + import com.iqser.red.service.search.v1.model.SearchRequest; import com.iqser.red.service.search.v1.model.SearchResult; import com.iqser.red.service.search.v1.resources.SearchResource; import com.iqser.red.service.search.v1.server.service.SearchService; -import lombok.RequiredArgsConstructor; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; +import lombok.RequiredArgsConstructor; @RestController @RequiredArgsConstructor @@ -18,8 +19,7 @@ public class SearchController implements SearchResource { public SearchResult getDossierStatus(@RequestBody SearchRequest searchRequest) { - return searchService.search( - searchRequest.getQueryString(), + return searchService.search(searchRequest.getQueryString(), searchRequest.getDossierTemplateIds(), searchRequest.getDossierIds(), searchRequest.getFileId(), @@ -30,8 +30,7 @@ public class SearchController implements SearchResource { searchRequest.getFileAttributes(), searchRequest.getPage(), searchRequest.getPageSize(), - searchRequest.isReturnSections() - ); + searchRequest.isReturnSections()); } } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java index 20bf125..85d655b 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/migration/MigrationStarterService.java @@ -33,11 +33,10 @@ public class MigrationStarterService { if (settings.isMigrateOnly()) { if (indexInformationService.hasIndexChanged()) { log.info("Index has changed and will be closed, dropped, recreated and all files will be indexed"); - indexingMessageReceiver.receiveIndexingRequest(IndexMessage.builder() - .messageType(IndexMessageType.DROP) - .build()); + indexingMessageReceiver.receiveIndexingRequest(IndexMessage.builder().messageType(IndexMessageType.DROP).build()); } System.exit(SpringApplication.exit(ctx, () -> 0)); } } + } \ No newline at end of file diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java index bf51fe5..eadeabb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocument.java @@ -1,12 +1,15 @@ package com.iqser.red.service.search.v1.server.model; -import lombok.*; - import java.io.Serializable; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @Data @Builder @@ -32,5 +35,4 @@ public class IndexDocument implements Serializable { @Builder.Default private List fileAttributes = new ArrayList<>(); - } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocumentUpdate.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocumentUpdate.java index caccd41..bb7a08a 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocumentUpdate.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexDocumentUpdate.java @@ -1,12 +1,12 @@ package com.iqser.red.service.search.v1.server.model; +import java.util.List; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; - @Data @Builder @NoArgsConstructor diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexSection.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexSection.java index b3124d7..da348a5 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexSection.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/IndexSection.java @@ -1,10 +1,14 @@ package com.iqser.red.service.search.v1.server.model; -import lombok.*; - import java.io.Serializable; import java.util.Set; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @Builder @NoArgsConstructor(access = AccessLevel.PRIVATE) @@ -16,4 +20,5 @@ public class IndexSection implements Serializable { private String text; private Set pages; private String headline; + } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Query.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Query.java index 3637629..4f037d0 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Query.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Query.java @@ -1,13 +1,13 @@ package com.iqser.red.service.search.v1.server.model; +import java.util.ArrayList; +import java.util.List; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.ArrayList; -import java.util.List; - @Data @Builder @NoArgsConstructor diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/SectionText.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/SectionText.java index 390bd22..1e51f63 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/SectionText.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/SectionText.java @@ -1,13 +1,13 @@ package com.iqser.red.service.search.v1.server.model; +import java.util.ArrayList; +import java.util.List; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.ArrayList; -import java.util.List; - @Data @Builder @NoArgsConstructor diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Text.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Text.java index e6831da..82ed8d9 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Text.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/model/Text.java @@ -1,13 +1,14 @@ package com.iqser.red.service.search.v1.server.model; -import com.dslplatform.json.CompiledJson; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - import java.util.ArrayList; import java.util.List; +import com.dslplatform.json.CompiledJson; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @CompiledJson @NoArgsConstructor diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java index 65cd49c..da4b033 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/IndexingMessageReceiver.java @@ -59,20 +59,21 @@ public class IndexingMessageReceiver { @SneakyThrows @RabbitHandler @RabbitListener(queues = INDEXING_QUEUE) - public void receiveIndexingRequest(Message message){ + public void receiveIndexingRequest(Message message) { var indexRequest = objectMapper.readValue(message.getBody(), IndexMessage.class); // This prevents from endless retries oom errors. - if(message.getMessageProperties().isRedelivered()){ - throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", indexRequest.getDossierId(), indexRequest.getFileId())); + if (message.getMessageProperties().isRedelivered()) { + throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", + indexRequest.getDossierId(), + indexRequest.getFileId())); } receiveIndexingRequest(indexRequest); } - public void receiveIndexingRequest(IndexMessage indexRequest) { log.info("Processing indexing request: {}", indexRequest); @@ -87,8 +88,12 @@ public class IndexingMessageReceiver { case UPDATE: fileStatus = fileStatusClient.getFileStatus(indexRequest.getDossierId(), indexRequest.getFileId()); dossier = dossierClient.getDossierById(indexRequest.getDossierId(), true, true); - documentUpdateService.updateDocument(indexRequest.getFileId(), fileStatus.getAssignee(), dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, fileStatus.getWorkflowStatus() - .name(), fileStatus.getFileAttributes()); + documentUpdateService.updateDocument(indexRequest.getFileId(), + fileStatus.getAssignee(), + dossier.getSoftDeletedTime() != null, + dossier.getArchivedTime() != null, + fileStatus.getWorkflowStatus().name(), + fileStatus.getFileAttributes()); log.info("Successfully updated {}", indexRequest); break; @@ -148,7 +153,16 @@ public class IndexingMessageReceiver { fileStatusProcessingUpdateClient.indexing(dossier.getId(), file.getId()); Text text = textStorageService.getText(dossier.getId(), file.getId()); - documentIndexService.indexDocument(dossier.getDossierTemplateId(), dossier.getId(), file.getId(), file.getFilename(), text, file.getAssignee(), dossier.getSoftDeletedTime() != null, dossier.getArchivedTime() != null, file.getWorkflowStatus(), file.getFileAttributes()); + documentIndexService.indexDocument(dossier.getDossierTemplateId(), + dossier.getId(), + file.getId(), + file.getFilename(), + text, + file.getAssignee(), + dossier.getSoftDeletedTime() != null, + dossier.getArchivedTime() != null, + file.getWorkflowStatus(), + file.getFileAttributes()); fileStatusProcessingUpdateClient.indexingSuccessful(dossier.getId(), file.getId()); log.info("Successfully indexed dossier {} file {}", dossier.getId(), file.getId()); } @@ -170,14 +184,12 @@ public class IndexingMessageReceiver { for (FileModel file : files) { try { log.info("Will add dossier {} file {} to index queue", dossierId, file.getId()); - rabbitTemplate.convertAndSend(INDEXING_QUEUE, objectMapper.writeValueAsString(IndexMessage.builder() - .messageType(IndexMessageType.INSERT) - .dossierId(dossierId) - .fileId(file.getId()) - .build()), message -> { - message.getMessageProperties().setPriority(99); - return message; - }); + rabbitTemplate.convertAndSend(INDEXING_QUEUE, + objectMapper.writeValueAsString(IndexMessage.builder().messageType(IndexMessageType.INSERT).dossierId(dossierId).fileId(file.getId()).build()), + message -> { + message.getMessageProperties().setPriority(99); + return message; + }); } catch (JsonProcessingException e) { throw new RuntimeException(e); } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java index 909d3c8..201571a 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/queue/MessagingConfiguration.java @@ -1,11 +1,12 @@ package com.iqser.red.service.search.v1.server.queue; -import lombok.RequiredArgsConstructor; import org.springframework.amqp.core.Queue; import org.springframework.amqp.core.QueueBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import lombok.RequiredArgsConstructor; + @Configuration @RequiredArgsConstructor public class MessagingConfiguration { @@ -20,11 +21,7 @@ public class MessagingConfiguration { @Bean public Queue indexingQueue() { - return QueueBuilder.durable(INDEXING_QUEUE) - .withArgument("x-dead-letter-exchange", "") - .withArgument("x-dead-letter-routing-key", INDEXING_DQL) - .maxPriority(2) - .build(); + return QueueBuilder.durable(INDEXING_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", INDEXING_DQL).maxPriority(2).build(); } @@ -52,5 +49,4 @@ public class MessagingConfiguration { return QueueBuilder.durable(DELETE_FROM_INDEX_DLQ).build(); } - } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java index 642ff57..c90b334 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentDeleteService.java @@ -1,16 +1,18 @@ package com.iqser.red.service.search.v1.server.service; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import lombok.RequiredArgsConstructor; +import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; + +import java.io.IOException; + import org.opensearch.action.delete.DeleteRequest; import org.opensearch.client.RequestOptions; import org.springframework.stereotype.Service; -import java.io.IOException; +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; +import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java index 33a1523..c1a8e0b 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentIndexService.java @@ -1,20 +1,6 @@ package com.iqser.red.service.search.v1.server.service; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.model.*; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; - -import io.micrometer.core.annotation.Timed; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.common.xcontent.XContentType; -import org.springframework.stereotype.Service; +import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; import java.io.IOException; import java.time.OffsetDateTime; @@ -23,7 +9,27 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.client.RequestOptions; +import org.opensearch.common.xcontent.XContentType; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.model.IndexDocument; +import com.iqser.red.service.search.v1.server.model.IndexFileAttribute; +import com.iqser.red.service.search.v1.server.model.IndexSection; +import com.iqser.red.service.search.v1.server.model.SectionArea; +import com.iqser.red.service.search.v1.server.model.SectionText; +import com.iqser.red.service.search.v1.server.model.Text; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @@ -36,9 +42,16 @@ public class DocumentIndexService { @Timed("redactmanager_indexDocument") - public void indexDocument(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, - boolean deleted, boolean archived, - WorkflowStatus workflowStatus, Map fileAttributes) { + public void indexDocument(String dossierTemplateId, + String dossierId, + String fileId, + String filename, + Text text, + String assignee, + boolean deleted, + boolean archived, + WorkflowStatus workflowStatus, + Map fileAttributes) { IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id(fileId); indexRequest.setRefreshPolicy(settings.getRefreshPolicy()); @@ -62,9 +75,16 @@ public class DocumentIndexService { } - private IndexDocument convert(String dossierTemplateId, String dossierId, String fileId, String filename, Text text, String assignee, - boolean deleted, boolean archived, - WorkflowStatus workflowStatus, Map fileAttributes) { + private IndexDocument convert(String dossierTemplateId, + String dossierId, + String fileId, + String filename, + Text text, + String assignee, + boolean deleted, + boolean archived, + WorkflowStatus workflowStatus, + Map fileAttributes) { return IndexDocument.builder() .dossierTemplateId(dossierTemplateId) diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java index 8910a37..58c2f05 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/DocumentUpdateService.java @@ -1,5 +1,17 @@ package com.iqser.red.service.search.v1.server.service; +import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.common.xcontent.XContentType; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; import com.iqser.red.service.search.v1.server.model.IndexDocumentUpdate; @@ -8,17 +20,6 @@ import com.iqser.red.service.search.v1.server.model.IndexFileAttribute; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; -import org.opensearch.action.update.UpdateRequest; -import org.opensearch.action.update.UpdateResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.common.xcontent.XContentType; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; @Service @RequiredArgsConstructor @@ -27,6 +28,7 @@ public class DocumentUpdateService { private final ObjectMapper objectMapper; private final ElasticsearchClient client; + @SneakyThrows @Timed("redactmanager_updateDocument") public void updateDocument(String fileId, String assignee, boolean deleted, boolean archived, String workflowStatus, Map fileAttributes) { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java index 4d29947..087ee01 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexCreatorService.java @@ -1,10 +1,9 @@ package com.iqser.red.service.search.v1.server.service; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import com.iqser.red.service.search.v1.server.utils.ResourceLoader; -import lombok.extern.slf4j.Slf4j; +import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; + +import java.io.IOException; + import org.opensearch.client.RequestOptions; import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.CreateIndexResponse; @@ -13,9 +12,12 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.springframework.stereotype.Service; -import java.io.IOException; +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.exception.IndexException; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.service.search.v1.server.utils.ResourceLoader; -import static com.iqser.red.service.search.v1.server.exception.IndexException.INDEX_EXISTS_ERROR; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service @@ -47,8 +49,7 @@ public class IndexCreatorService { .put("number_of_replicas", settings.getNumberOfReplicas()) .put("index.mapping.nested_objects.limit", settings.getNumberOfNestedObjectLimit()); - CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME).settings(settingsBuilder.build()) - .mapping(indexMapping, XContentType.JSON); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME).settings(settingsBuilder.build()).mapping(indexMapping, XContentType.JSON); try { CreateIndexResponse response = client.indices().create(createIndexRequest, RequestOptions.DEFAULT); diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java index 29df883..82c69ef 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexDeleteService.java @@ -1,10 +1,7 @@ package com.iqser.red.service.search.v1.server.service; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.exception.IndexException; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; +import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; + import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.RequestOptions; @@ -12,7 +9,12 @@ import org.opensearch.client.indices.CloseIndexRequest; import org.opensearch.common.unit.TimeValue; import org.springframework.stereotype.Service; -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.exception.IndexException; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; @Slf4j @Service diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java index 18198fd..208b0a6 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/IndexInformationService.java @@ -35,7 +35,10 @@ public class IndexInformationService { return true; } String fileHash = generateIndexConfigurationHash(); - log.info("Hash from database {} (updated {}) and hash from file {}", indexInformationFromDatabase.getIndexConfigurationHash(), indexInformationFromDatabase.getUpdateDate(), fileHash); + log.info("Hash from database {} (updated {}) and hash from file {}", + indexInformationFromDatabase.getIndexConfigurationHash(), + indexInformationFromDatabase.getUpdateDate(), + fileHash); if (StringUtils.equals(indexInformationFromDatabase.getIndexConfigurationHash(), fileHash)) { return false; diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java index 27f2483..de757cb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/SearchService.java @@ -1,5 +1,37 @@ package com.iqser.red.service.search.v1.server.service; +import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; +import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.RequestOptions; +import org.opensearch.common.text.Text; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.InnerHitBuilder; +import org.opensearch.index.query.NestedQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.springframework.stereotype.Service; + import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.search.v1.model.MatchedDocument; import com.iqser.red.service.search.v1.model.MatchedSection; @@ -13,26 +45,6 @@ import io.micrometer.core.annotation.Timed; import io.micrometer.core.instrument.util.StringUtils; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.lucene.search.join.ScoreMode; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.common.text.Text; -import org.opensearch.index.query.*; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.fetch.subphase.FetchSourceContext; -import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.iqser.red.service.search.v1.server.exception.IndexException.FAILED_TO_SEARCH; -import static com.iqser.red.service.search.v1.server.service.IndexCreatorService.INDEX_NAME; @Slf4j @Service @@ -43,30 +55,35 @@ public class SearchService { @Timed("redactmanager_search") - public SearchResult search( - String queryString, - List dossierTemplateIds, - List dossierIds, - String fileId, - String assignee, - boolean includeDeletedDossiers, - boolean includeArchivedDossiers, - String workflowStatus, - Map fileAttributes, - int page, - int pageSize, - boolean returnSections - ) { + public SearchResult search(String queryString, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + int page, + int pageSize, + boolean returnSections) { Query query = QueryStringConverter.convert(queryString); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query( - convertQuery(query, dossierTemplateIds, dossierIds, fileId, assignee, - includeDeletedDossiers, includeArchivedDossiers, workflowStatus, fileAttributes, returnSections)) + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(convertQuery(query, + dossierTemplateIds, + dossierIds, + fileId, + assignee, + includeDeletedDossiers, + includeArchivedDossiers, + workflowStatus, + fileAttributes, + returnSections)) .from(getPageOrDefault(page) * getPageSizeOrDefault(pageSize)) .size(getPageSizeOrDefault(pageSize)) - .fetchSource(new String[]{"dossierId", "dossierTemplateId", "dossierDeleted", "dossierArchived", "filename", - "fileId", "assignee", "dossierStatus", "workflowStatus", "fileAttributes"}, new String[]{"sections"}) + .fetchSource(new String[]{"dossierId", "dossierTemplateId", "dossierDeleted", "dossierArchived", "filename", "fileId", "assignee", "dossierStatus", "workflowStatus", "fileAttributes"}, + new String[]{"sections"}) .highlighter(new HighlightBuilder().field("sections.text").field("filename").field("fileAttributes.value").highlighterType("fvh")) .trackScores(true); @@ -88,18 +105,16 @@ public class SearchService { } - private QueryBuilder convertQuery( - Query query, - List dossierTemplateIds, - List dossierIds, - String fileId, - String assignee, - boolean includeDeletedDossiers, - boolean includeArchivedDossiers, - String workflowStatus, - Map fileAttributes, - boolean returnSections - ) { + private QueryBuilder convertQuery(Query query, + List dossierTemplateIds, + List dossierIds, + String fileId, + String assignee, + boolean includeDeletedDossiers, + boolean includeArchivedDossiers, + String workflowStatus, + Map fileAttributes, + boolean returnSections) { BoolQueryBuilder entireQuery = QueryBuilders.boolQuery(); BoolQueryBuilder sectionsQueries = QueryBuilders.boolQuery(); @@ -110,10 +125,7 @@ public class SearchService { QueryBuilder filenamePhraseQuery = QueryBuilders.matchPhraseQuery("filename", must.toLowerCase(Locale.ROOT)).queryName("filename." + must); QueryBuilder fileAttributesPhraseQuery = QueryBuilders.matchPhraseQuery("fileAttributes.value", must.toLowerCase(Locale.ROOT)).queryName("fileAttributes." + must); - QueryBuilder filenameOrTextMustQuery = QueryBuilders.boolQuery() - .should(textPhraseQuery) - .should(filenamePhraseQuery) - .should(fileAttributesPhraseQuery); + QueryBuilder filenameOrTextMustQuery = QueryBuilders.boolQuery().should(textPhraseQuery).should(filenamePhraseQuery).should(fileAttributesPhraseQuery); entireQuery.must(filenameOrTextMustQuery); sectionsQueries.should(textPhraseQuery); } @@ -130,11 +142,13 @@ public class SearchService { if (returnSections) { NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery("sections", sectionsQueries, ScoreMode.Avg) - .innerHit(new InnerHitBuilder().setSize(100).setFetchSourceContext(new FetchSourceContext(true, new String[]{"sections.headline", "sections.sectionNumber", "sections.pages"}, new String[]{"sections.text"}))); + .innerHit(new InnerHitBuilder().setSize(100) + .setFetchSourceContext(new FetchSourceContext(true, + new String[]{"sections.headline", "sections.sectionNumber", "sections.pages"}, + new String[]{"sections.text"}))); entireQuery.should(nestedQuery); } - BoolQueryBuilder filterQuery = QueryBuilders.boolQuery(); if (dossierTemplateIds != null && !dossierTemplateIds.isEmpty()) { @@ -208,9 +222,7 @@ public class SearchService { private SearchResult convert(SearchResponse response, Query query) { return SearchResult.builder() - .matchedDocuments(Arrays.stream(response.getHits().getHits()) - .map(hit -> convertSearchHit(hit, query)) - .collect(Collectors.toList())) + .matchedDocuments(Arrays.stream(response.getHits().getHits()).map(hit -> convertSearchHit(hit, query)).collect(Collectors.toList())) .maxScore(response.getHits().getMaxScore()) .total(response.getHits().getTotalHits().value) .build(); @@ -224,9 +236,7 @@ public class SearchService { .map(match -> match.contains("fileAttributes.") ? match.replace("fileAttributes.", "") : match) .collect(Collectors.toSet()); - Set unmatchedTerms = Stream.concat(query.getMusts().stream(), query.getShoulds().stream()) - .filter(term -> !matchesTerms.contains(term)) - .collect(Collectors.toSet()); + Set unmatchedTerms = Stream.concat(query.getMusts().stream(), query.getShoulds().stream()).filter(term -> !matchesTerms.contains(term)).collect(Collectors.toSet()); MatchedDocument.MatchedDocumentBuilder matchedDocumentBuilder = MatchedDocument.builder() .score(hit.getScore()) @@ -243,18 +253,14 @@ public class SearchService { .highlights(hit.getHighlightFields() .entrySet() .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> Arrays.stream(e.getValue().getFragments()) - .map(Text::string) - .collect(Collectors.toSet())))) + .collect(Collectors.toMap(Map.Entry::getKey, e -> Arrays.stream(e.getValue().getFragments()).map(Text::string).collect(Collectors.toSet())))) .matchedTerms(matchesTerms) .unmatchedTerms(unmatchedTerms); if (hit.getInnerHits() != null) { SearchHits sectionHits = hit.getInnerHits().get("sections"); - matchedDocumentBuilder.matchedSections(Arrays.stream(sectionHits.getHits()) - .map(innerHit -> convertInnerHit(innerHit)) - .collect(Collectors.toList())) + matchedDocumentBuilder.matchedSections(Arrays.stream(sectionHits.getHits()).map(innerHit -> convertInnerHit(innerHit)).collect(Collectors.toList())) .containsAllMatchedSections(sectionHits.getTotalHits().value == sectionHits.getHits().length); } @@ -264,6 +270,7 @@ public class SearchService { private Map convertFileAttributes(Object fileAttributesSourceMap) { + Map fileAttributes = new HashMap<>(); if (fileAttributesSourceMap != null) { @@ -285,6 +292,7 @@ public class SearchService { .build(); } + private int getPageSizeOrDefault(int pageSize) { if (pageSize <= 0) { @@ -302,5 +310,4 @@ public class SearchService { return page; } - } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java index ab20b25..2474feb 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/service/TextStorageService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.search.v1.server.service; +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.search.v1.server.model.Text; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; @@ -9,8 +11,6 @@ import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - @Slf4j @Service @RequiredArgsConstructor diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java index 5289066..fef07a2 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/ElasticsearchSettings.java @@ -1,13 +1,14 @@ package com.iqser.red.service.search.v1.server.settings; -import lombok.Data; +import java.util.ArrayList; +import java.util.List; + import org.opensearch.action.support.WriteRequest; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import java.util.ArrayList; -import java.util.List; +import lombok.Data; @Data @Primary @@ -34,4 +35,5 @@ public class ElasticsearchSettings { * ES refresh policy for write requests to use. Used in tests to wait for completion of write requests. */ private WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.NONE; + } diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java index 73a9abf..44b0c7e 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/settings/SearchServiceSettings.java @@ -1,10 +1,11 @@ package com.iqser.red.service.search.v1.server.settings; -import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import lombok.Data; + @Data @Primary @Configuration diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/QueryStringConverter.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/QueryStringConverter.java index da4283e..be50f27 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/QueryStringConverter.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/QueryStringConverter.java @@ -1,12 +1,13 @@ package com.iqser.red.service.search.v1.server.utils; -import com.iqser.red.service.search.v1.server.model.Query; -import io.micrometer.core.instrument.util.StringUtils; -import lombok.experimental.UtilityClass; - import java.util.ArrayList; import java.util.List; +import com.iqser.red.service.search.v1.server.model.Query; + +import io.micrometer.core.instrument.util.StringUtils; +import lombok.experimental.UtilityClass; + @UtilityClass public class QueryStringConverter { diff --git a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java index 231c711..a0803a7 100644 --- a/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java +++ b/search-service-v1/search-service-server-v1/src/main/java/com/iqser/red/service/search/v1/server/utils/ResourceLoader.java @@ -1,7 +1,5 @@ package com.iqser.red.service.search.v1.server.utils; -import lombok.experimental.UtilityClass; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -10,6 +8,8 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; + @UtilityClass public class ResourceLoader { @@ -19,12 +19,11 @@ public class ResourceLoader { if (resource == null) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath); } - try (InputStream is = resource.openStream(); - InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); - BufferedReader br = new BufferedReader(isr)) { + try (InputStream is = resource.openStream(); InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) { return br.lines().collect(Collectors.joining("\n")); } catch (IOException e) { throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e); } } + } diff --git a/search-service-v1/search-service-server-v1/src/main/resources/index/mapping.json b/search-service-v1/search-service-server-v1/src/main/resources/index/mapping.json index ff86c6b..ea8bc51 100644 --- a/search-service-v1/search-service-server-v1/src/main/resources/index/mapping.json +++ b/search-service-v1/search-service-server-v1/src/main/resources/index/mapping.json @@ -29,7 +29,7 @@ "dossierDeleted": { "type": "boolean" }, - "fileAttributes" : { + "fileAttributes": { "type": "nested", "include_in_parent": true, "properties": { diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java index 160ab8a..7212026 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/AbstractElasticsearchIntegrationTest.java @@ -1,11 +1,5 @@ package com.iqser.red.service.search.v1.server.service; -import com.iqser.red.service.search.v1.server.Application; -import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; -import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; -import com.iqser.red.storage.commons.StorageAutoConfiguration; -import com.iqser.red.storage.commons.service.StorageService; - import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -25,11 +19,15 @@ import org.springframework.test.context.junit4.SpringRunner; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; +import com.iqser.red.service.search.v1.server.Application; +import com.iqser.red.service.search.v1.server.client.ElasticsearchClient; +import com.iqser.red.service.search.v1.server.settings.ElasticsearchSettings; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; @ComponentScan @RunWith(SpringRunner.class) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, - properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS}) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = {AbstractElasticsearchIntegrationTest.WAIT_FOR_WRITE_REQUESTS}) @ContextConfiguration(initializers = {AbstractElasticsearchIntegrationTest.Initializer.class}) @EnableFeignClients(basePackageClasses = AbstractElasticsearchIntegrationTest.TestConfiguration.class) @DirtiesContext @@ -37,8 +35,8 @@ public abstract class AbstractElasticsearchIntegrationTest { public static final String WAIT_FOR_WRITE_REQUESTS = "elasticsearch.refreshPolicy=IMMEDIATE"; - static class Initializer implements ApplicationContextInitializer { + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { var esContainer = new ElasticsearchContainer(DockerImageName.parse("elasticsearch:7.17.2").asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch")); @@ -50,9 +48,9 @@ public abstract class AbstractElasticsearchIntegrationTest { TestPropertyValues.of( // "elasticsearch.cluster.hosts[0]=" + host, - "elasticsearch.port=" + port - ).applyTo(configurableApplicationContext.getEnvironment()); + "elasticsearch.port=" + port).applyTo(configurableApplicationContext.getEnvironment()); } + } @Autowired @@ -66,8 +64,7 @@ public abstract class AbstractElasticsearchIntegrationTest { public static class TestConfiguration { @Bean - public IndexCreatorService indexCreationService(ElasticsearchClient elasticsearchClient, - ElasticsearchSettings elasticsearchSettings) { + public IndexCreatorService indexCreationService(ElasticsearchClient elasticsearchClient, ElasticsearchSettings elasticsearchSettings) { return new IndexCreatorService(elasticsearchClient, elasticsearchSettings); @@ -77,6 +74,7 @@ public abstract class AbstractElasticsearchIntegrationTest { @Bean @Primary public StorageService inmemoryStorage() { + return new FileSystemBackedStorageService(); } diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java index bada8a5..5c3c02e 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/FileSystemBackedStorageService.java @@ -1,13 +1,5 @@ package com.iqser.red.service.search.v1.server.service; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.iqser.red.commons.jackson.ObjectMapperFactory; -import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; -import com.iqser.red.storage.commons.service.StorageService; -import lombok.SneakyThrows; -import org.apache.commons.io.IOUtils; -import org.springframework.core.io.InputStreamResource; - import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -18,13 +10,26 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.io.IOUtils; +import org.springframework.core.io.InputStreamResource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.commons.jackson.ObjectMapperFactory; +import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; +import com.iqser.red.storage.commons.service.StorageService; + +import lombok.SneakyThrows; + public class FileSystemBackedStorageService implements StorageService { private final Map dataMap = new HashMap<>(); + public FileSystemBackedStorageService() { + } + @SneakyThrows @Override public InputStreamResource getObject(String objectId) { @@ -37,48 +42,62 @@ public class FileSystemBackedStorageService implements StorageService { } + @Override public void deleteObject(String objectId) { + dataMap.remove(objectId); } + @Override public boolean objectExists(String objectId) { + return dataMap.containsKey(objectId); } + @Override public void init() { } + @Override @SneakyThrows public void storeJSONObject(String objectId, T any) { + File tempFile = File.createTempFile("test", ".tmp"); getMapper().writeValue(new FileOutputStream(tempFile), any); dataMap.put(objectId, tempFile); } + private ObjectMapper getMapper() { - return ObjectMapperFactory.create(); + + return ObjectMapperFactory.create(); } + @Override @SneakyThrows public T readJSONObject(String objectId, Class clazz) { + if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) { throw new StorageObjectDoesNotExist("Stored object not found"); } return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz); } + public List listPaths() { + return new ArrayList<>(dataMap.keySet()); } public List listFilePaths() { + return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList()); } @@ -86,6 +105,7 @@ public class FileSystemBackedStorageService implements StorageService { @Override @SneakyThrows public void storeObject(String objectId, InputStream stream) { + File tempFile = File.createTempFile("test", ".tmp"); try (var fileOutputStream = new FileOutputStream(tempFile)) { @@ -97,9 +117,11 @@ public class FileSystemBackedStorageService implements StorageService { public void clearStorage() { + this.dataMap.forEach((k, v) -> { v.delete(); }); this.dataMap.clear(); } + } diff --git a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java index 39b1e5e..0a8b1cb 100644 --- a/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java +++ b/search-service-v1/search-service-server-v1/src/test/java/com/iqser/red/service/search/v1/server/service/SearchTest.java @@ -73,8 +73,26 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", text, "UserId2", false, false, WorkflowStatus.NEW, Map.of("F2Key", "F2Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); // Act & Assert 1 SearchResult result = searchService.search("F1Value", null, null, null, null, false, false, null, null, 0, 10, false); @@ -90,10 +108,30 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value", "F2Key", "F2Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value", "F2Key", "F2Value")); // Act & Assert 1 - SearchResult result = searchService.search("S-Metolachlor", List.of("template1"), List.of("dossierId1"), "fileId1", "UserId", false, false, WorkflowStatus.NEW.name(), Map.of("F1Key", "F1Value"), 0, 10, false); + SearchResult result = searchService.search("S-Metolachlor", + List.of("template1"), + List.of("dossierId1"), + "fileId1", + "UserId", + false, + false, + WorkflowStatus.NEW.name(), + Map.of("F1Key", "F1Value"), + 0, + 10, + false); assertThat(result.getMatchedDocuments().size()).isEqualTo(1); System.out.println(result); @@ -110,8 +148,26 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", text, "UserId2", false, false, WorkflowStatus.NEW, Map.of("F2Key", "F2Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); // Act & Assert 1 SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); @@ -150,8 +206,26 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", text, "UserId2", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); // Act & Assert 1 SearchResult result = searchService.search("S-Metolachlor", null, null, null, "UserId2", false, false, null, null, 0, 10, false); @@ -178,8 +252,26 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", text, "UserId2", false, false, WorkflowStatus.NEW, Map.of("F2Key", "F2Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); // Act & Assert 1 SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, WorkflowStatus.APPROVED.name(), null, 0, 10, false); @@ -223,8 +315,26 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", text, "UserId2", false, false, WorkflowStatus.NEW, Map.of("F2Key", "F2Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); // Act & Assert 1 SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, true, false, null, null, 0, 10, false); @@ -279,8 +389,26 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource = new ClassPathResource("files/Text2.json"); Text text = objectMapper.readValue(textResource.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", text, "UserId2", false, false, WorkflowStatus.NEW, Map.of("F2Key", "F2Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-1.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06-2.pdf", + text, + "UserId2", + false, + false, + WorkflowStatus.NEW, + Map.of("F2Key", "F2Value")); // Act & Assert 1 SearchResult result = searchService.search("S-Metolachlor", null, null, null, null, false, false, null, Map.of("F1Key", "F1Value"), 0, 10, false); @@ -339,8 +467,26 @@ public class SearchTest extends AbstractElasticsearchIntegrationTest { ClassPathResource textResource2 = new ClassPathResource("files/Text2.json"); Text text2 = objectMapper.readValue(textResource2.getInputStream(), Text.class); - documentIndexService.indexDocument("template1", "dossierId1", "fileId1", "Single Study - Oral (Gavage) Mouse.pdf", text, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); - documentIndexService.indexDocument("template2", "dossierId2", "fileId2", "S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", text2, "UserId", false, false, WorkflowStatus.NEW, Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template1", + "dossierId1", + "fileId1", + "Single Study - Oral (Gavage) Mouse.pdf", + text, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); + documentIndexService.indexDocument("template2", + "dossierId2", + "fileId2", + "S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", + text2, + "UserId", + false, + false, + WorkflowStatus.NEW, + Map.of("F1Key", "F1Value")); // Act & Assert 1 SearchResult result = searchService.search("S-Metolachlor", null, List.of("dossierId1", "dossierId2"), null, null, false, false, null, null, 0, 10, false);