Change Pattern

enable caching in build

See merge request fforesight/layout-parser!84

.gitlab-ci.yml

@@ -10,14 +10,12 @@ deploy:
   script:
     - echo "Building with gradle version ${BUILDVERSION}"
     - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
     - echo "BUILDVERSION=$BUILDVERSION" >> version.env
   artifacts:
     reports:
       dotenv: version.env
   rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
-pmd:
-  allow_failure: true

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -196,12 +196,6 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public float getTextHeightNoPadding() {
-
-        return textPositions.get(0).getHeightDir();
-    }
-
-
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public float getTextHeight() {
@@ -240,7 +234,6 @@ public class TextPositionSequence implements CharSequence {
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public String getFontStyle() {
-
         if (textPositions.get(0).getFontName() == null) {
             return "standard";
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocuMineBlockificationService.java

@@ -61,8 +61,8 @@ public class DocuMineBlockificationService {
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
             boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
-            boolean splitByOtherFontAndOtherY = prev != null && Math.abs(prev.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * 0.2 && (word.getFontStyle()
-                    .contains("bold") && !prev.getFontStyle().contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
+            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle()
+                    .contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
 
             Matcher matcher = pattern.matcher(chunkWords.stream().collect(Collectors.joining(" ")).toString());
             boolean startsOnSameX = Math.abs(minX - word.getMinXDirAdj()) < 5 && matcher.matches();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java

@@ -4,9 +4,11 @@ import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.HashSet;
+import java.util.Set;
 
+import org.apache.pdfbox.cos.COSDictionary;
 import org.apache.pdfbox.cos.COSName;
-import org.apache.pdfbox.pdfwriter.compress.CompressParameters;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
 import org.apache.pdfbox.pdmodel.PDPage;
@@ -38,6 +40,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class ViewerDocumentService {
 
+
     private static final String LAYER_NAME = "Layout grid";
     private static final int FONT_SIZE = 10;
     public static final float LINE_WIDTH = 1f;
@@ -51,7 +54,8 @@ public class ViewerDocumentService {
         LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
         // PDDocument.save() is very slow, since it actually traverses the entire pdf and writes a new one.
         // If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
-        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, layerVisibilityDefaultValue);
+        Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
+        PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
         PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);
 
         for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
@@ -64,8 +68,7 @@ public class ViewerDocumentService {
             // e.g. not escaped matrix transformations.
             escapePreviousContents(pdDocument, pdPage);
 
-            VisualizationsOnPage visualizationsOnPage = layoutGrid.getVisualizationsPerPages()
-                    .get(pageNumber);
+            VisualizationsOnPage visualizationsOnPage = layoutGrid.getVisualizationsPerPages().get(pageNumber);
             assert pageNumber == visualizationsOnPage.getPageNumber();
             // We need to append to the content stream, otherwise the content could be overlapped by following content.
             try (var contentStream = new PDPageContentStream(pdDocument, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
@@ -99,11 +102,11 @@ public class ViewerDocumentService {
                     contentStream.setFont(font, FONT_SIZE);
                     contentStream.beginText();
                     Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(),
-                                                   (float) textDeRotationMatrix.getShearX(),
-                                                   (float) textDeRotationMatrix.getShearY(),
-                                                   (float) textDeRotationMatrix.getScaleY(),
-                                                   (float) placedText.lineStart().getX(),
-                                                   (float) placedText.lineStart().getY());
+                            (float) textDeRotationMatrix.getShearX(),
+                            (float) textDeRotationMatrix.getShearY(),
+                            (float) textDeRotationMatrix.getScaleY(),
+                            (float) placedText.lineStart().getX(),
+                            (float) placedText.lineStart().getY());
                     textMatrix.translate(-((font.getStringWidth(placedText.text()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
                     contentStream.setTextMatrix(textMatrix);
                     contentStream.showText(placedText.text());
@@ -112,9 +115,12 @@ public class ViewerDocumentService {
                 contentStream.restoreGraphicsState();
                 contentStream.endMarkedContent();
             }
+            dictionariesToUpdate.add(pdPage.getCOSObject());
+            dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
         }
+        dictionariesToUpdate.add(pdDocument.getDocumentInformation().getCOSObject());
 //        dictionariesToUpdate.add(pdDocument.getDocument().getTrailer());
-        pdDocument.save(outputStream, CompressParameters.NO_COMPRESSION);
+        pdDocument.saveIncremental(outputStream, dictionariesToUpdate);
     }
 
 
@@ -139,7 +145,7 @@ public class ViewerDocumentService {
     }
 
 
-    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, boolean layerVisibilityDefaultValue) {
+    private static PDOptionalContentGroup addLayerToDocument(PDDocument pdDocument, Set<COSDictionary> dictionariesToUpdate, boolean layerVisibilityDefaultValue) {
 
         PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
         PDOptionalContentProperties ocprops = catalog.getOCProperties();
@@ -155,6 +161,7 @@ public class ViewerDocumentService {
             ocprops.addGroup(layer);
         }
         ocprops.setGroupEnabled(layer, layerVisibilityDefaultValue);
+        dictionariesToUpdate.add(catalog.getCOSObject());
         return layer;
     }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionSequenceComparator.java

@@ -28,13 +28,15 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPosit
  *
  * @author Ben Litchfield
  */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
-
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
+{
     @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
+    {
         // only compare text that is in the same direction
         int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0) {
+        if (cmp1 != 0)
+        {
             return cmp1;
         }
 
@@ -46,19 +48,25 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe
         float pos2YBottom = pos2.getMaxYDirAdj();
 
         // note that the coordinates have been adjusted so 0,0 is in upper left
-        float pos1YTop = pos1YBottom - pos1.getTextHeightNoPadding();
-        float pos2YTop = pos2YBottom - pos2.getTextHeightNoPadding();
+        float pos1YTop = pos1YBottom - pos1.getTextHeight();
+        float pos2YTop = pos2YBottom - pos2.getTextHeight();
 
         float yDifference = Math.abs(pos1YBottom - pos2YBottom);
 
         // we will do a simple tolerance comparison
-        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
+        if (yDifference < .1 ||
+                pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
+                pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
+        {
             return Float.compare(x1, x2);
-        } else if (pos1YBottom < pos2YBottom) {
+        }
+        else if (pos1YBottom < pos2YBottom)
+        {
             return -1;
-        } else {
+        }
+        else
+        {
             return 1;
         }
     }
-
 }

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -21,6 +21,7 @@ dependencies {
 
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
     implementation("com.knecon.fforesight:tenant-commons:0.14.0")
+    implementation("com.knecon.fforesight:tracing:0.5.0")
     implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
     implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
     implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")

layoutparser-service/layoutparser-service-server/src/main/resources/application.yml

@@ -5,6 +5,8 @@ tenant-user-management-service.url: "http://tenant-user-management-service:8080/
 
 fforesight.tenants.remote: true
 
+logging.pattern.level: "%5p [${spring.application.name},namespace=${management.namespace},traceID=%X{traceId:-NONE},spanID=%X{spanId:-NONE}]"
+
 server:
   port: 8080
 
@@ -32,6 +34,14 @@ management:
     prometheus.enabled: ${monitoring.enabled:false}
     health.enabled: true
   endpoints.web.exposure.include: prometheus, health
+  namespace: ${NAMESPACE:default}
+  tracing:
+    enabled: ${TRACING_ENABLED:false}
+    sampling:
+      probability: ${TRACING_PROBABILITY:1.0}
+  otlp:
+    tracing:
+      endpoint: ${OTLP_ENDPOINT:http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces}
 
 
 storage: