hotfix: test to add fonts

Update dependency com.iqser.red.commons:storage-commons to v2.29.0 (main)

See merge request fforesight/layout-parser!17

.gitlab-ci.yml

@@ -1,4 +1,21 @@
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
-    file: 'ci-templates/gradle_java.yml'
+    file: 'ci-templates/gradle_java.yml'
+
+deploy:
+  stage: deploy
+  tags:
+    - dind
+  script:
+    - echo "Building with gradle version ${BUILDVERSION}"
+    - gradle -Pversion=${BUILDVERSION} publish
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  artifacts:
+    reports:
+      dotenv: version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -5,18 +5,18 @@ plugins {
 
 description = "layoutparser-service-processor"
 
-val jacksonVersion = "2.15.0-rc2"
-val pdfBoxVersion = "3.0.0-alpha2"
+val jacksonVersion = "2.15.2"
+val pdfBoxVersion = "3.0.0-RC1"
 
 dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0") {
+    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.124.0") {
         exclude("org.springframework.boot", "spring-boot-starter-security")
         exclude("org.springframework.boot", "spring-boot-starter-validation")
     }
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:2.29.0")
 
     implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     implementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -5,6 +5,9 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.pdfbox.Loader;
@@ -16,14 +19,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -37,10 +41,11 @@ public class LayoutParsingStorageService {
 
     public PDDocument getOriginFile(String storageId) throws IOException {
 
-        try (var originDocumentInputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (var originDocumentInputStream = getObject(storageId)) {
             File tempFile = createTempFile("document", ".pdf");
             try (var tempFileOutputStream = new FileOutputStream(tempFile)) {
                 IOUtils.copy(originDocumentInputStream, tempFileOutputStream);
+                originDocumentInputStream.close();
             }
             return Loader.loadPDF(tempFile, MemoryUsageSetting.setupMixed(67108864L));
         }
@@ -49,19 +54,22 @@ public class LayoutParsingStorageService {
 
     public ImageServiceResponse getImagesFile(String storageId) throws IOException {
 
-        try (InputStream inputStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
+        try (InputStream inputStream = getObject(storageId)) {
 
-            return objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            ImageServiceResponse imageServiceResponse = objectMapper.readValue(inputStream, ImageServiceResponse.class);
+            inputStream.close();
+            return imageServiceResponse;
         }
     }
 
 
     public TableServiceResponse getTablesFile(String storageId) throws IOException {
 
-        try (var tableClassificationStream = storageService.getObject(TenantContext.getTenantId(), storageId).getInputStream()) {
-
-            return objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+        try (var tableClassificationStream = getObject(storageId)) {
 
+            TableServiceResponse tableServiceResponse = objectMapper.readValue(tableClassificationStream, TableServiceResponse.class);
+            tableClassificationStream.close();
+            return tableServiceResponse;
         }
     }
 
@@ -117,4 +125,13 @@ public class LayoutParsingStorageService {
         storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
     }
 
+    @SneakyThrows
+    private InputStream getObject(String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
+        Path path = Paths.get(tempFile.getPath());
+        return Files.newInputStream(path, StandardOpenOption.DELETE_ON_CLOSE);
+    }
+
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -44,7 +44,10 @@ public class DocuMineBlockificationService {
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractPageBlock> chunkBlockList1 = new ArrayList<>();
 
-        float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
+        float minX = 1000;
+        float maxX = 0;
+        float minY = 1000;
+        float maxY = 0;
         TextPositionSequence prev = null;
 
         boolean wasSplitted = false;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/PdfVisualisationUtility.java

@@ -12,7 +12,6 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
@@ -72,7 +71,7 @@ public class PdfVisualisationUtility {
 
         contentStream.beginText();
         contentStream.newLineAtOffset((float) location.getX(), (float) location.getY());
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 10);
+        contentStream.setFont(PDType1Font.HELVETICA, 10);
         contentStream.showText(string);
         contentStream.endText();
         contentStream.close();

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -4,25 +4,25 @@ plugins {
     application
     id("com.knecon.fforesight.java-conventions")
     id("org.springframework.boot") version "3.1.2"
-    id("io.spring.dependency-management") version "1.1.0"
-    id("org.sonarqube") version "4.2.1.3168"
+    id("io.spring.dependency-management") version "1.1.2"
+    id("org.sonarqube") version "4.3.0.3225"
     id("io.freefair.lombok") version "8.1.0"
 //    id("org.graalvm.buildtools.native") version "0.9.23"
 }
 
 description = "layoutparser-service-server"
 
-val jacksonVersion = "2.15.0-rc2"
+val jacksonVersion = "2.15.2"
 val pdfBoxVersion = "3.0.0-alpha2"
 
 dependencies {
     implementation(project(":layoutparser-service-processor"))
     implementation(project(":layoutparser-service-internal-api"))
 
-    implementation("com.iqser.red.commons:storage-commons:2.1.0")
+    implementation("com.iqser.red.commons:storage-commons:2.29.0")
     implementation("com.knecon.fforesight:tenant-commons:0.10.0")
     implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
-    implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
+    implementation("com.amazonaws:aws-java-sdk-s3:1.12.526")
     implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
 
@@ -30,8 +30,8 @@ dependencies {
     testImplementation(project(":layoutparser-service-internal-api"))
 
     testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
-    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
-    testImplementation("com.iqser.red.commons:jackson-commons:1.0.0")
+    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.124.0")
+    testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
     testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
@@ -70,6 +70,33 @@ sonarqube {
 }
 
 tasks.named<BootBuildImage>("bootBuildImage") {
+
+//    environment.put("BP_ACCEPT_EULA","Y")
+////    println("--------------------" + layout.projectDirectory.file( "src/main/resources/Aptfile").toString());
+//    var aptfilePath = layout.projectDirectory.file( "src/main/resources/Aptfile").toString() // make sure this path is valid because there will be no log or exception if not
+//    bindings.add("${aptfilePath}:/workspace/Aptfile:ro")
+//    buildpacks.set(listOf("ghcr.io/fagiani/buildpacks/fagiani_apt@sha256:6471c8c70f32b749e29f65ae562ac0339fecad26aa9217628c00a6c31f197dae", "urn:cnb:builder:paketo-buildpacks/java"))
+
+
+//    environment.put("BP_LOG_LEVEL","DEBUG")
+//    var aptfilePath = layout.projectDirectory.file( "src/main/resources/Aptfile").toString() // make sure this path is valid because there will be no log or exception if not
+//    bindings.add("${aptfilePath}:/workspace/Aptfile:ro")
+//    var shfilePath = layout.projectDirectory.file( "src/main/resources/buildpack-run.sh").toString() // make sure this path is valid because there will be no log or exception if not
+//    bindings.add("${shfilePath}:/workspace/buildpack-run.sh:ro")
+//    buildpacks.set(listOf("ghcr.io/fagiani/buildpacks/fagiani_run@sha256:5c75bf651072e1ad345aa148433b8cdd373e97f9b29523ceaeaf1c39f1691985", "ghcr.io/fagiani/buildpacks/fagiani_apt@sha256:6471c8c70f32b749e29f65ae562ac0339fecad26aa9217628c00a6c31f197dae","urn:cnb:builder:paketo-buildpacks/java"))
+
+    var fontsfilePath = layout.projectDirectory.file( "src/main/resources/fonts.tar.gz").toString() // make sure this path is valid because there will be no log or exception if not
+    bindings.add("${fontsfilePath}:/workspace/fonts.tar.gz:ro")
+
+    var shfilePath = layout.projectDirectory.file( "src/main/resources/buildpack-run.sh").toString() // make sure this path is valid because there will be no log or exception if not
+    bindings.add("${shfilePath}:/workspace/buildpack-run.sh:ro")
+    buildpacks.set(listOf("ghcr.io/fagiani/buildpacks/fagiani_run@sha256:5c75bf651072e1ad345aa148433b8cdd373e97f9b29523ceaeaf1c39f1691985", "urn:cnb:builder:paketo-buildpacks/java"))
+
+
+
+
+
+
     imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
     if (project.hasProperty("buildbootDockerHostNetwork")) {
         network.set("host")

layoutparser-service/layoutparser-service-server/src/main/resources/Aptfile

@@ -0,0 +1,14 @@
+# you can list packages
+ghostscript
+cabextract
+xfonts-utils
+fonts-liberation
+lmodern
+ttf-mscorefonts-installer
+
+
+# or include links to specific .deb files
+# http://ftp.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.8_all.deb
+
+# or add custom apt repos (only required if using packages outside of the standard Ubuntu APT repositories)
+# :repo:deb http://cz.archive.ubuntu.com/ubuntu artful main universe

layoutparser-service/layoutparser-service-server/src/main/resources/buildpack-run.sh

@@ -0,0 +1 @@
+tar -xf fonts.tar.gz

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java

@@ -13,7 +13,6 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -36,14 +35,14 @@ public class FileSystemBackedStorageService implements StorageService {
 
     @SneakyThrows
     @Override
-    public InputStreamResource getObject(String tenantId, String objectId) {
+    public void downloadTo(String tenantId, String objectId, File destinationFile) {
 
         var res = dataMap.get(objectId);
         if (res == null) {
             throw new StorageObjectDoesNotExist(new RuntimeException());
         }
-        return new InputStreamResource(new FileInputStream(res));
 
+        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
     }
 
 

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}