Marmelator/layout-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 289 Wiki Activity

Merge branch 'RED-6725' into 'main'

Browse Source 
Red 6725

See merge request fforesight/layout-parser!4
This commit is contained in:
Kilian Schüttler 2023-07-28 17:46:40 +02:00
commit 5561dd5e95
9 changed files with 86 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
build.gradle.kts
View File

@ -1,127 +0,0 @@
import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
java
id("org.springframework.boot") version "3.1.2"
id("io.spring.dependency-management") version "1.1.2"
id("org.sonarqube") version "4.3.0.3225"
pmd
checkstyle
jacoco
}
group = "com.knecon.fforesight"
java.sourceCompatibility = JavaVersion.VERSION_17
configurations {
compileOnly {
extendsFrom(configurations.annotationProcessor.get())
}
}
tasks.pmdMain{
pmd.ruleSetFiles = files("${projectDir}/config/pmd/pmd.xml")
}
tasks.pmdTest {
pmd.ruleSetFiles = files("${projectDir}/config/pmd/test_pmd.xml")
}
tasks.jacocoTestReport {
reports {
xml.required.set(false)
csv.required.set(false)
html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
}
}
repositories {
mavenLocal()
mavenCentral()
maven {
url = uri("https://nexus.knecon.com/repository/gindev/");
credentials {
username = providers.gradleProperty("mavenUser").getOrNull();
password = providers.gradleProperty("mavenPassword").getOrNull();
}
}
}
tasks.named<BootBuildImage>("bootBuildImage") {
imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
if (project.hasProperty("buildbootDockerHostNetwork")) {
network.set("host")
}
docker {
if (project.hasProperty("buildbootDockerHostNetwork")) {
bindHostToBuilder.set(true)
}
verboseLogging.set(true)
publishRegistry {
username.set(providers.gradleProperty("mavenUser").getOrNull())
password.set(providers.gradleProperty("mavenPassword").getOrNull())
email.set(providers.gradleProperty("mavenEmail").getOrNull())
url.set("https://nexus.knecon.com:5001/")
}
}
}
extra["springCloudVersion"] = "2022.0.4"
extra["testcontainersVersion"] = "1.18.3"
dependencies {
implementation("jakarta.json:jakarta.json-api:2.1.2")
implementation("co.elastic.clients:elasticsearch-java:8.6.2")
implementation("org.springframework.boot:spring-boot-starter-actuator")
implementation("org.springframework.boot:spring-boot-starter-amqp")
implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.cloud:spring-cloud-starter-openfeign")
implementation("com.iqser.red.commons:storage-commons:2.15.0")
implementation("com.knecon.fforesight:keycloak-commons:0.18.0")
implementation("com.knecon.fforesight:swagger-commons:0.5.0")
compileOnly("org.projectlombok:lombok")
developmentOnly("org.springframework.boot:spring-boot-devtools")
annotationProcessor("org.springframework.boot:spring-boot-configuration-processor")
annotationProcessor("org.projectlombok:lombok")
testImplementation("org.springframework.boot:spring-boot-starter-test")
testImplementation("org.springframework.amqp:spring-rabbit-test")
testImplementation("org.testcontainers:elasticsearch:1.18.3")
testAnnotationProcessor("org.projectlombok:lombok")
}
dependencyManagement {
imports {
mavenBom("org.testcontainers:testcontainers-bom:${property("testcontainersVersion")}")
mavenBom("org.springframework.cloud:spring-cloud-dependencies:${property("springCloudVersion")}")
}
}
tasks.withType<Test> {
useJUnitPlatform()
reports {
junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
}
}
sonarqube {
properties {
providers.gradleProperty("sonarToken").getOrNull()?.let { property("sonar.login", it) }
property("sonar.host.url", "https://sonarqube.knecon.com")
}
}
tasks.test {
finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
}
tasks.jacocoTestReport {
dependsOn(tasks.test) // tests are required to run before generating the report
reports {
xml.required.set(true)
csv.required.set(false)
}
}

41
buildSrc/src/main/kotlin/com.knecon.fforesight.java-conventions.gradle.kts
View File

@ -1,24 +1,16 @@
plugins {
java
`java-library`
`maven-publish`
pmd
checkstyle
jacoco
}
group = "com.knecon.fforesight"
version = "0.1-SNAPSHOT"
java.sourceCompatibility = JavaVersion.VERSION_17
java.targetCompatibility = JavaVersion.VERSION_17
tasks.jacocoTestReport {
reports {
xml.required.set(false)
csv.required.set(false)
html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
}
}
tasks.pmdMain {
pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
}
@ -29,6 +21,9 @@ tasks.pmdTest {
tasks.named<Test>("test") {
useJUnitPlatform()
reports {
junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
}
}
tasks.test {
@ -40,14 +35,38 @@ tasks.jacocoTestReport {
reports {
xml.required.set(true)
csv.required.set(false)
html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
}
}
allprojects {
publishing {
publications {
create<MavenPublication>(name) {
from(components["java"])
}
}
repositories {
maven {
url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
credentials {
username = providers.gradleProperty("mavenUser").getOrNull();
password = providers.gradleProperty("mavenPassword").getOrNull();
}
}
}
}
}
java {
withJavadocJar()
}
repositories {
mavenLocal()
mavenCentral()
maven {
url = uri("https://nexus.knecon.com/repository/gindev/");
url = uri("https://nexus.knecon.com/repository/gindev/")
credentials {
username = providers.gradleProperty("mavenUser").getOrNull();
password = providers.gradleProperty("mavenPassword").getOrNull();

5
layoutparser-service/layoutparser-service-processor/build.gradle.kts
View File

@ -3,6 +3,8 @@ plugins {
id("io.freefair.lombok") version "8.1.0"
}
description = "layoutparser-service-processor"
dependencies {
implementation(project(":layoutparser-service-internal-api"))
@ -18,7 +20,4 @@ dependencies {
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
}
description = "layoutparser-service-processor"

24
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java
View File

@ -15,11 +15,7 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
@ -90,26 +86,6 @@ public class LayoutParsingStorageService {
}
public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
layoutParsingRequest.textBlockFileStorageId(),
DocumentTextData[].class);
DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
layoutParsingRequest.positionBlockFileStorageId(),
DocumentPositionData[].class);
DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);
return DocumentData.builder()
.documentStructure(tableOfContentsData)
.documentPositions(atomicPositionBlockData)
.documentTextData(documentTextDataBlockData)
.documentPages(documentPageData)
.build();
}
private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();

40
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessagingConfiguration.java
View File

@ -1,40 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.queue;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import lombok.RequiredArgsConstructor;
@Configuration
@RequiredArgsConstructor
public class MessagingConfiguration {
@Bean
public Queue layoutparsingRequestQueue() {
return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingResponseQueue() {
return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingDLQ() {
return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
}
}

18
layoutparser-service/layoutparser-service-server/build.gradle.kts
View File

@ -10,15 +10,18 @@ plugins {
// id("org.graalvm.buildtools.native") version "0.9.23"
}
description = "layoutparser-service-server"
dependencies {
implementation(project(":layoutparser-service-processor"))
implementation(project(":layoutparser-service-internal-api"))
implementation("com.iqser.red.commons:storage-commons:2.1.0")
implementation("com.knecon.fforesight:tenant-commons:0.10.0")
implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
//implementation("com.azure.spring:spring-cloud-azure-native-configuration:4.0.0-beta.1")
// for integration testing only
testImplementation(project(":layoutparser-service-internal-api"))
@ -32,12 +35,6 @@ dependencies {
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
}
description = "layoutparser-service-server"
java {
withJavadocJar()
}
/*
//TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something
//kinda weird behavior, since the buildArgs below seem to have no effect at all
@ -61,6 +58,13 @@ tasks.named<Pmd>("pmdAotTest") {
}
*/
sonarqube {
properties {
providers.gradleProperty("sonarToken").getOrNull()?.let { property("sonar.login", it) }
property("sonar.host.url", "https://sonarqube.knecon.com")
}
}
tasks.named<BootBuildImage>("bootBuildImage") {
imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
if (project.hasProperty("buildbootDockerHostNetwork")) {

2
layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/Application.java
View File

@ -10,7 +10,7 @@ import org.springframework.context.annotation.Import;
import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.queue.MessagingConfiguration;
import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})

10
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/queue/MessageHandler.java → layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java
View File

@ -1,7 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.queue;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
package com.knecon.fforesight.service.layoutparser.server.queue;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
@ -12,6 +9,7 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@ -30,7 +28,7 @@ public class MessageHandler {
@RabbitHandler
@RabbitListener(queues = LAYOUT_PARSING_REQUEST_QUEUE)
@RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
@SneakyThrows
public void receiveLayoutParsingRequest(Message message) {
@ -49,7 +47,7 @@ public class MessageHandler {
public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
rabbitTemplate.convertAndSend(LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
}
}

38
layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java Normal file
View File

@ -0,0 +1,38 @@
package com.knecon.fforesight.service.layoutparser.server.queue;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import lombok.RequiredArgsConstructor;
@Configuration
@RequiredArgsConstructor
public class MessagingConfiguration {
@Bean
public Queue layoutparsingRequestQueue() {
return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingResponseQueue() {
return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingDLQ() {
return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
}
}