Merge branch 'RED-6725' into 'main'

Red 6725

See merge request fforesight/layout-parser!4

buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts

@@ -1,24 +1,16 @@
 plugins {
-    java
+    `java-library`
+    `maven-publish`
     pmd
     checkstyle
     jacoco
 }
 
 group = "com.knecon.fforesight"
-version = "0.1-SNAPSHOT"
 
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
-tasks.jacocoTestReport {
-    reports {
-        xml.required.set(false)
-        csv.required.set(false)
-        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
-    }
-}
-
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }
@@ -29,6 +21,9 @@ tasks.pmdTest {
 
 tasks.named<Test>("test") {
     useJUnitPlatform()
+    reports {
+        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
+    }
 }
 
 tasks.test {
@@ -40,14 +35,38 @@ tasks.jacocoTestReport {
     reports {
         xml.required.set(true)
         csv.required.set(false)
+        html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
     }
 }
 
+allprojects {
+    publishing {
+        publications {
+            create<MavenPublication>(name) {
+                from(components["java"])
+            }
+        }
+        repositories {
+            maven {
+                url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+                credentials {
+                    username = providers.gradleProperty("mavenUser").getOrNull();
+                    password = providers.gradleProperty("mavenPassword").getOrNull();
+                }
+            }
+        }
+    }
+}
+
+java {
+    withJavadocJar()
+}
+
 repositories {
     mavenLocal()
     mavenCentral()
     maven {
-        url = uri("https://nexus.knecon.com/repository/gindev/");
+        url = uri("https://nexus.knecon.com/repository/gindev/")
         credentials {
             username = providers.gradleProperty("mavenUser").getOrNull();
             password = providers.gradleProperty("mavenPassword").getOrNull();

gradle.properties.kts

@@ -1 +1 @@
-version = 0.1.0
+version = 0.1-SNAPSHOT

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -3,10 +3,15 @@ plugins {
     id("io.freefair.lombok") version "8.1.0"
 }
 
+description = "layoutparser-service-processor"
+
 dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
     implementation("com.iqser.red.commons:storage-commons:2.1.0")
 
@@ -15,7 +20,4 @@ dependencies {
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
     implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
 }
-
-description = "layoutparser-service-processor"

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -15,11 +15,7 @@ import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
@@ -90,26 +86,6 @@ public class LayoutParsingStorageService {
     }
 
 
-    public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
-
-        DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
-        DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.textBlockFileStorageId(),
-                DocumentTextData[].class);
-        DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
-                layoutParsingRequest.positionBlockFileStorageId(),
-                DocumentPositionData[].class);
-        DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);
-
-        return DocumentData.builder()
-                .documentStructure(tableOfContentsData)
-                .documentPositions(atomicPositionBlockData)
-                .documentTextData(documentTextDataBlockData)
-                .documentPages(documentPageData)
-                .build();
-    }
-
-
     private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
 
         File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java

@@ -1,40 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.queue;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
-
-import org.springframework.amqp.core.Queue;
-import org.springframework.amqp.core.QueueBuilder;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-import lombok.RequiredArgsConstructor;
-
-@Configuration
-@RequiredArgsConstructor
-public class MessagingConfiguration {
-
-    @Bean
-    public Queue layoutparsingRequestQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingResponseQueue() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
-                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
-    }
-
-
-    @Bean
-    public Queue layoutparsingDLQ() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
-    }
-
-}

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -1,23 +1,26 @@
 import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 
 plugins {
+    application
     id("com.knecon.fforesight.java-conventions")
     id("org.springframework.boot") version "3.1.2"
     id("io.spring.dependency-management") version "1.1.0"
     id("org.sonarqube") version "4.2.1.3168"
     id("io.freefair.lombok") version "8.1.0"
-    //id("org.graalvm.buildtools.native") version "0.9.23"
+//    id("org.graalvm.buildtools.native") version "0.9.23"
 }
 
+description = "layoutparser-service-server"
+
 dependencies {
     implementation(project(":layoutparser-service-processor"))
+    implementation(project(":layoutparser-service-internal-api"))
 
     implementation("com.iqser.red.commons:storage-commons:2.1.0")
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-
     implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
     implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
-
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
 
     // for integration testing only
     testImplementation(project(":layoutparser-service-internal-api"))
@@ -30,23 +33,17 @@ dependencies {
     testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
     testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
     testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
-    testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
-}
-
-description = "layoutparser-service-server"
-
-java {
-    withJavadocJar()
 }
 
 /*
-TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something
-kinda weird behavior, since the buildArgs below seem to have no effect at all
-https://stackoverflow.com/questions/72770461/graalvm-native-image-can-not-compile-logback-dependencies
+//TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something
+//kinda weird behavior, since the buildArgs below seem to have no effect at all
+//https://stackoverflow.com/questions/72770461/graalvm-native-image-can-not-compile-logback-dependencies
 
+We traced the issue to storage-commons and there to azure blob storage, this is today an open bug with azure spring
+https://github.com/Azure/azure-sdk-for-java/issues/36001
 
-// AOT seems to be the name of the generated classes for native images
-// They are added as a SourceSet, and therefore checkstyle and pmd try to run on it
+//AOT is springs ahead of time compilation, it is a code analysis which generates new code, therefore checkstyle and pmd can be disabled on them.
 tasks.named<Checkstyle>("checkstyleAot") {
     enabled = false
 }
@@ -59,28 +56,15 @@ tasks.named<Pmd>("pmdAot") {
 tasks.named<Pmd>("pmdAotTest") {
     enabled = false
 }
+*/
 
-graalvmNative {
-    binaries {
-        all {
-            resources.autodetect()
-            buildArgs.add("--verbose")
-            buildArgs.add("--add-opens=java.base/java.nio=ALL-UNNAMED")
-            buildArgs.add("--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED")
-            buildArgs.add("--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED")
-            buildArgs.add("--trace-class-initialization=ch.qos.logback.classic.Logger")
-            buildArgs.add("--trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase\$Worker")
-            buildArgs.add("--initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback")
-            buildArgs.add("--initialize-at-run-time=io.netty")
-        }
-
+sonarqube {
+    properties {
+        providers.gradleProperty("sonarToken").getOrNull()?.let { property("sonar.login", it) }
+        property("sonar.host.url", "https://sonarqube.knecon.com")
     }
 }
 
-nativeBuild {
-    buildArgs("-H:ReflectionConfigurationFiles=../../../src/main/resources/reflection-config.json")
-}
-*/
 tasks.named<BootBuildImage>("bootBuildImage") {
     imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
     if (project.hasProperty("buildbootDockerHostNetwork")) {

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java

@@ -10,7 +10,7 @@ import org.springframework.context.annotation.Import;
 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
-import com.knecon.fforesight.service.layoutparser.processor.queue.MessagingConfiguration;
+import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -1,7 +1,4 @@
-package com.knecon.fforesight.service.layoutparser.processor.queue;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
+package com.knecon.fforesight.service.layoutparser.server.queue;
 
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -12,6 +9,7 @@ import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 
@@ -30,7 +28,7 @@ public class MessageHandler {
 
 
     @RabbitHandler
-    @RabbitListener(queues = LAYOUT_PARSING_REQUEST_QUEUE)
+    @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
     @SneakyThrows
     public void receiveLayoutParsingRequest(Message message) {
 
@@ -49,7 +47,7 @@ public class MessageHandler {
 
     public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
 
-        rabbitTemplate.convertAndSend(LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
     }
 
 }

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java

@@ -0,0 +1,38 @@
+package com.knecon.fforesight.service.layoutparser.server.queue;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
+
+import lombok.RequiredArgsConstructor;
+
+@Configuration
+@RequiredArgsConstructor
+public class MessagingConfiguration {
+
+    @Bean
+    public Queue layoutparsingRequestQueue() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingResponseQueue() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+
+    @Bean
+    public Queue layoutparsingDLQ() {
+
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
+    }
+
+}