From b5523842ddc0033df87fe6baf7f75971664f8011 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 25 Oct 2024 19:20:27 +0200 Subject: [PATCH] fix short word snuggification --- .../BBoxSnuggificationService.java | 42 +++++++++++++++---- .../StrokeWidthCalculator.java | 13 ++++-- .../WritableOcrResultFactory.java | 7 +--- .../src/test/resources/application.yml | 4 +- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/BBoxSnuggificationService.java b/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/BBoxSnuggificationService.java index 07aad92..8fae568 100644 --- a/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/BBoxSnuggificationService.java +++ b/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/BBoxSnuggificationService.java @@ -13,6 +13,7 @@ import com.knecon.fforesight.service.ocr.v1.api.model.QuadPoint; import com.sun.jna.Pointer; import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import net.sourceforge.lept4j.Leptonica1; import net.sourceforge.lept4j.Numa; @@ -23,6 +24,7 @@ import net.sourceforge.lept4j.util.LeptUtils; * This class attempts to shrink the BBox of a word to match the exact height of the word. This is only attempted for horizontal or vertical words. Any askew text is left as is. */ @Slf4j +@UtilityClass public class BBoxSnuggificationService { public static final int PIXEL_COUNT_THRESHOLD = 2; // minimum active pixel count per row for shrinking to stop @@ -32,12 +34,13 @@ public class BBoxSnuggificationService { private enum Operation { HORIZONTAL, VERTICAL, + BOTH, NONE } @SneakyThrows - public static Optional snuggify(Pix pageImage, DocumentWord origin, AffineTransform resultToImageTransform) { + public Optional snuggify(Pix pageImage, DocumentWord origin, AffineTransform resultToImageTransform) { if (pageImage == null) { return Optional.empty(); @@ -60,11 +63,17 @@ public class BBoxSnuggificationService { return Optional.empty(); } + if (!StrokeWidthCalculator.wordImageHasMinimumPixelDensity(wordImage)) { + return Optional.empty(); + } + Optional snugBox = switch (operation) { case HORIZONTAL -> snuggifyY(wordImage, originTransformed.getBounds2D()); case VERTICAL -> snuggifyX(wordImage, originTransformed.getBounds2D()); + case BOTH -> snuggifyBoth(wordImage, originTransformed); default -> Optional.empty(); }; + LeptUtils.disposePix(wordImage); AffineTransform imageToResultTransform = resultToImageTransform.createInverse(); @@ -74,20 +83,35 @@ public class BBoxSnuggificationService { } - private static Operation determineOperation(DocumentWord origin, QuadPoint.Direction direction, double remainingAngle, QuadPoint originTransformed) { + private Optional snuggifyBoth(Pix wordImage, QuadPoint originTransformed) { + + Optional snugY = snuggifyY(wordImage, originTransformed.getBounds2D()); + Optional snugX = snuggifyX(wordImage, originTransformed.getBounds2D()); + if (snugY.isPresent() && snugX.isPresent()) { + return Optional.of(snugY.get().createIntersection(snugX.get()).getBounds2D()); + } else if (snugY.isPresent()) { + return snugY; + } else { + return snugX; + } + } + + + private Operation determineOperation(DocumentWord origin, QuadPoint.Direction direction, double remainingAngle, QuadPoint originTransformed) { Operation operation = Operation.NONE; - if (((direction.equals(QuadPoint.Direction.RIGHT) || direction.equals(QuadPoint.Direction.LEFT)) && remainingAngle < INDIVIDUAL_ANGLE_THRESHOLD) // - || (origin.getContent().length() < 4 || Math.abs(originTransformed.getAngle()) < AVERAGE_ANGLE_THRESHOLD * 3)) { + if (((direction.equals(QuadPoint.Direction.RIGHT) || direction.equals(QuadPoint.Direction.LEFT)) && remainingAngle < INDIVIDUAL_ANGLE_THRESHOLD)) { operation = Operation.HORIZONTAL; } else if ((direction.equals(QuadPoint.Direction.UP) || direction.equals(QuadPoint.Direction.DOWN)) && remainingAngle < INDIVIDUAL_ANGLE_THRESHOLD) { operation = Operation.VERTICAL; + } else if ((origin.getContent().length() < 4 || Math.abs(originTransformed.getAngle()) < AVERAGE_ANGLE_THRESHOLD * 3)) { + return Operation.BOTH; } return operation; } - private static Optional snuggifyX(Pix wordImage, Rectangle2D origin) { + private Optional snuggifyX(Pix wordImage, Rectangle2D origin) { Numa colCounts = Leptonica1.pixCountPixelsByColumn(wordImage); int start = 0; @@ -112,7 +136,7 @@ public class BBoxSnuggificationService { } - private static Optional snuggifyY(Pix wordImage, Rectangle2D origin) { + private Optional snuggifyY(Pix wordImage, Rectangle2D origin) { int start = 0; int end = wordImage.h - 1; @@ -135,7 +159,7 @@ public class BBoxSnuggificationService { } - private static int pixCountPerRow(int row, Pix pix) { + private int pixCountPerRow(int row, Pix pix) { IntBuffer result = IntBuffer.allocate(1); int success = Leptonica1.pixCountPixelsInRow(pix, row, result, null); @@ -147,7 +171,7 @@ public class BBoxSnuggificationService { } - private static int pixCountPerColumn(int column, Numa colCounts) { + private int pixCountPerColumn(int column, Numa colCounts) { if (column > colCounts.n) { throw new IndexOutOfBoundsException("column " + column + " is out of bounds for column count " + colCounts.n); @@ -159,7 +183,7 @@ public class BBoxSnuggificationService { } - public static boolean canBeSnuggified(DocumentPage resultPage, AffineTransform imageTransform) { + public boolean canBeSnuggified(DocumentPage resultPage, AffineTransform imageTransform) { double averageAngle = resultPage.getWords() .stream() diff --git a/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/StrokeWidthCalculator.java b/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/StrokeWidthCalculator.java index a1c5333..820a199 100644 --- a/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/StrokeWidthCalculator.java +++ b/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/StrokeWidthCalculator.java @@ -3,13 +3,9 @@ package com.knecon.fforesight.service.ocr.processor.service.imageprocessing; import static net.sourceforge.lept4j.ILeptonica.L_THIN_FG; import java.io.Closeable; -import java.io.IOException; import java.nio.IntBuffer; -import org.springframework.stereotype.Service; - import lombok.AccessLevel; -import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; import net.sourceforge.lept4j.Leptonica1; import net.sourceforge.lept4j.Pix; @@ -19,6 +15,7 @@ import net.sourceforge.lept4j.util.LeptUtils; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class StrokeWidthCalculator implements Closeable { + public static final double MINIMUM_PIXEL_DENSITY = 0.05; Sela thinningSel = Leptonica1.selaMakeThinSets(1, 0); @@ -46,6 +43,14 @@ public class StrokeWidthCalculator implements Closeable { } + public static boolean wordImageHasMinimumPixelDensity(Pix wordImage) { + + IntBuffer pixelCount = IntBuffer.allocate(1); + Leptonica1.pixCountPixels(wordImage, pixelCount, null); + return (double) pixelCount.get(0) / (wordImage.w * wordImage.h) >= MINIMUM_PIXEL_DENSITY; + } + + public boolean hasLargerStrokeWidth(Pix pix, double strokeWidth, double threshold) { int roundedStrokeWidth = (int) Math.round(strokeWidth); diff --git a/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/WritableOcrResultFactory.java b/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/WritableOcrResultFactory.java index c6fdb16..66cbb16 100644 --- a/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/WritableOcrResultFactory.java +++ b/azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/WritableOcrResultFactory.java @@ -3,7 +3,6 @@ package com.knecon.fforesight.service.ocr.processor.visualizations; import java.awt.geom.AffineTransform; import java.awt.geom.Line2D; import java.awt.geom.Rectangle2D; -import java.nio.IntBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -37,6 +36,7 @@ import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.FontS import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.ImageProcessingPipeline; import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.ImageProcessingSupervisor; import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.BBoxSnuggificationService; +import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.StrokeWidthCalculator; import com.knecon.fforesight.service.ocr.processor.visualizations.fonts.FontMetricsProvider; import com.knecon.fforesight.service.ocr.processor.visualizations.fonts.FontStyle; import com.knecon.fforesight.service.ocr.processor.visualizations.fonts.Type0FontMetricsProvider; @@ -185,10 +185,7 @@ public class WritableOcrResultFactory { continue; } - IntBuffer pixelCount = IntBuffer.allocate(1); - Leptonica1.pixCountPixels(wordImage, pixelCount, null); - - if (pixelCount.get(0) > 3) { + if (StrokeWidthCalculator.wordImageHasMinimumPixelDensity(wordImage)) { fontStyleDetector.add(textPosition, wordImage, textPosition.getFontSizeByHeight()); } diff --git a/azure-ocr-service/azure-ocr-service-server/src/test/resources/application.yml b/azure-ocr-service/azure-ocr-service-server/src/test/resources/application.yml index 0615285..5ea1a41 100644 --- a/azure-ocr-service/azure-ocr-service-server/src/test/resources/application.yml +++ b/azure-ocr-service/azure-ocr-service-server/src/test/resources/application.yml @@ -7,7 +7,9 @@ azure: logging.type: ${LOGGING_TYPE:CONSOLE} -ocrService.sendStatusUpdates: false +ocrService: + sendStatusUpdates: false + debug: true management: endpoint: