From ad03ef192205e05664333d66b8d6934ea834ae8d Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 28 Jul 2023 16:15:24 +0200 Subject: [PATCH] move queue to server package, so i can easily import processor as a library for redaction-service --- ...con.fforesight.java-conventions.gradle.kts | 17 ++++---- .../build.gradle.kts | 2 +- .../LayoutParsingStorageService.java | 24 ----------- .../queue/MessagingConfiguration.java | 40 ------------------- .../build.gradle.kts | 18 +++++---- .../layoutparser/server/Application.java | 2 +- .../server}/queue/MessageHandler.java | 10 ++--- .../server/queue/MessagingConfiguration.java | 38 ++++++++++++++++++ 8 files changed, 64 insertions(+), 87 deletions(-) delete mode 100644 layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java rename layoutparser-service/{layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor => layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server}/queue/MessageHandler.java (80%) create mode 100644 layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java diff --git a/buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts index a64e24a..3c5edf4 100644 --- a/buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts @@ -11,14 +11,6 @@ version = "0.1-SNAPSHOT" java.sourceCompatibility = JavaVersion.VERSION_17 java.targetCompatibility = JavaVersion.VERSION_17 -tasks.jacocoTestReport { - reports { - xml.required.set(false) - csv.required.set(false) - html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml")) - } -} - tasks.pmdMain { pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml") } @@ -29,6 +21,9 @@ tasks.pmdTest { tasks.named("test") { useJUnitPlatform() + reports { + junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit")) + } } tasks.test { @@ -40,9 +35,15 @@ tasks.jacocoTestReport { reports { xml.required.set(true) csv.required.set(false) + html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml")) } } + +java { + withJavadocJar() +} + repositories { mavenLocal() mavenCentral() diff --git a/layoutparser-service/layoutparser-service-processor/build.gradle.kts b/layoutparser-service/layoutparser-service-processor/build.gradle.kts index a71c8f0..69546cb 100644 --- a/layoutparser-service/layoutparser-service-processor/build.gradle.kts +++ b/layoutparser-service/layoutparser-service-processor/build.gradle.kts @@ -18,7 +18,7 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2") implementation("org.springframework.boot:spring-boot-starter-web:3.0.6") - implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6") + } description = "layoutparser-service-processor" diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java index 52e210c..0b14dee 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java @@ -15,11 +15,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.storage.commons.service.StorageService; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; -import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse; @@ -90,26 +86,6 @@ public class LayoutParsingStorageService { } - public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException { - - DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class); - DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(), - layoutParsingRequest.textBlockFileStorageId(), - DocumentTextData[].class); - DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(), - layoutParsingRequest.positionBlockFileStorageId(), - DocumentPositionData[].class); - DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class); - - return DocumentData.builder() - .documentStructure(tableOfContentsData) - .documentPositions(atomicPositionBlockData) - .documentTextData(documentTextDataBlockData) - .documentPages(documentPageData) - .build(); - } - - private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException { File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile(); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java deleted file mode 100644 index 52c6c1c..0000000 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.knecon.fforesight.service.layoutparser.processor.queue; - -import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE; -import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ; -import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; - -import org.springframework.amqp.core.Queue; -import org.springframework.amqp.core.QueueBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import lombok.RequiredArgsConstructor; - -@Configuration -@RequiredArgsConstructor -public class MessagingConfiguration { - - @Bean - public Queue layoutparsingRequestQueue() { - - return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)// - .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); - } - - - @Bean - public Queue layoutparsingResponseQueue() { - - return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)// - .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build(); - } - - - @Bean - public Queue layoutparsingDLQ() { - - return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build(); - } - -} diff --git a/layoutparser-service/layoutparser-service-server/build.gradle.kts b/layoutparser-service/layoutparser-service-server/build.gradle.kts index bf10605..cdd4332 100644 --- a/layoutparser-service/layoutparser-service-server/build.gradle.kts +++ b/layoutparser-service/layoutparser-service-server/build.gradle.kts @@ -10,15 +10,18 @@ plugins { // id("org.graalvm.buildtools.native") version "0.9.23" } +description = "layoutparser-service-server" + dependencies { implementation(project(":layoutparser-service-processor")) + implementation(project(":layoutparser-service-internal-api")) implementation("com.iqser.red.commons:storage-commons:2.1.0") implementation("com.knecon.fforesight:tenant-commons:0.10.0") implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2") implementation("com.amazonaws:aws-java-sdk-s3:1.12.514") + implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6") - //implementation("com.azure.spring:spring-cloud-azure-native-configuration:4.0.0-beta.1") // for integration testing only testImplementation(project(":layoutparser-service-internal-api")) @@ -32,12 +35,6 @@ dependencies { testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6") } -description = "layoutparser-service-server" - -java { - withJavadocJar() -} - /* //TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something //kinda weird behavior, since the buildArgs below seem to have no effect at all @@ -61,6 +58,13 @@ tasks.named("pmdAotTest") { } */ +sonarqube { + properties { + providers.gradleProperty("sonarToken").getOrNull()?.let { property("sonar.login", it) } + property("sonar.host.url", "https://sonarqube.knecon.com") + } +} + tasks.named("bootBuildImage") { imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}") if (project.hasProperty("buildbootDockerHostNetwork")) { diff --git a/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java b/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java index 55478bb..75cfabd 100644 --- a/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java +++ b/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java @@ -10,7 +10,7 @@ import org.springframework.context.annotation.Import; import com.amazonaws.services.s3.model.metrics.MetricsConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; -import com.knecon.fforesight.service.layoutparser.processor.queue.MessagingConfiguration; +import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration; import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration; @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class}) diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessageHandler.java b/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java similarity index 80% rename from layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessageHandler.java rename to layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java index a834a25..3f89dd8 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessageHandler.java +++ b/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java @@ -1,7 +1,4 @@ -package com.knecon.fforesight.service.layoutparser.processor.queue; - -import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE; -import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE; +package com.knecon.fforesight.service.layoutparser.server.queue; import org.springframework.amqp.AmqpRejectAndDontRequeueException; import org.springframework.amqp.core.Message; @@ -12,6 +9,7 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; @@ -30,7 +28,7 @@ public class MessageHandler { @RabbitHandler - @RabbitListener(queues = LAYOUT_PARSING_REQUEST_QUEUE) + @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE) @SneakyThrows public void receiveLayoutParsingRequest(Message message) { @@ -49,7 +47,7 @@ public class MessageHandler { public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) { - rabbitTemplate.convertAndSend(LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent); + rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent); } } diff --git a/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java b/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java new file mode 100644 index 0000000..4fa503a --- /dev/null +++ b/layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java @@ -0,0 +1,38 @@ +package com.knecon.fforesight.service.layoutparser.server.queue; + +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.QueueBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames; + +import lombok.RequiredArgsConstructor; + +@Configuration +@RequiredArgsConstructor +public class MessagingConfiguration { + + @Bean + public Queue layoutparsingRequestQueue() { + + return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)// + .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build(); + } + + + @Bean + public Queue layoutparsingResponseQueue() { + + return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)// + .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build(); + } + + + @Bean + public Queue layoutparsingDLQ() { + + return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build(); + } + +}