Marmelator/azure-ocr-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 35 Wiki Activity

Compare commits

base: Marmelator:main
Branches Tags
Marmelator:main
Marmelator:serverbuild-test
Marmelator:release/0.6.x
Marmelator:0.6.x
Marmelator:RED-9353
Marmelator:0.26.0
Marmelator:0.25.0
Marmelator:0.24.0
Marmelator:0.23.0
Marmelator:0.22.0
Marmelator:ghostscript-test3
Marmelator:ghostscript-test2
Marmelator:ghostscript-test
Marmelator:0.21.0
Marmelator:0.20.0
Marmelator:0.19.0
Marmelator:0.18.0
Marmelator:0.17.0
Marmelator:0.16.0
Marmelator:0.15.0
Marmelator:0.14.0
Marmelator:0.13.0
Marmelator:0.12.0
Marmelator:0.11.0
Marmelator:0.6.2
Marmelator:0.6.1
Marmelator:0.10.0
Marmelator:0.9.0
Marmelator:0.8.0
Marmelator:0.7.0
Marmelator:0.6.0
Marmelator:0.6.0-RED-99-chris.1
Marmelator:0.6.0-RED-99.1
Marmelator:0.5.0
Marmelator:0.4.0
Marmelator:0.3.0
Marmelator:0.2.0
Marmelator:ocrDomX
Marmelator:0.1.0
Marmelator:0.0.0
..
compare: Marmelator:0.20.0
Branches Tags
Marmelator:main
Marmelator:serverbuild-test
Marmelator:release/0.6.x
Marmelator:0.6.x
Marmelator:RED-9353
Marmelator:0.26.0
Marmelator:0.25.0
Marmelator:0.24.0
Marmelator:0.23.0
Marmelator:0.22.0
Marmelator:ghostscript-test3
Marmelator:ghostscript-test2
Marmelator:ghostscript-test
Marmelator:0.21.0
Marmelator:0.20.0
Marmelator:0.19.0
Marmelator:0.18.0
Marmelator:0.17.0
Marmelator:0.16.0
Marmelator:0.15.0
Marmelator:0.14.0
Marmelator:0.13.0
Marmelator:0.12.0
Marmelator:0.11.0
Marmelator:0.6.2
Marmelator:0.6.1
Marmelator:0.10.0
Marmelator:0.9.0
Marmelator:0.8.0
Marmelator:0.7.0
Marmelator:0.6.0
Marmelator:0.6.0-RED-99-chris.1
Marmelator:0.6.0-RED-99.1
Marmelator:0.5.0
Marmelator:0.4.0
Marmelator:0.3.0
Marmelator:0.2.0
Marmelator:ocrDomX
Marmelator:0.1.0
Marmelator:0.0.0

No commits in common. "main" and "0.20.0" have entirely different histories.
main ... 0.20.0

45 changed files with 184 additions and 947 deletions
Show Stats Expand all files Collapse all files

25
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/AzureAnalyzeResult.java Normal file
View File

@ -0,0 +1,25 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
import java.util.ArrayList;
import java.util.List;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Getter
@Builder
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class AzureAnalyzeResult {
@Builder.Default
List<KeyValuePair> keyValuePairs = new ArrayList<>();
@Builder.Default
List<TextRegion> handWrittenText = new ArrayList<>();
@Builder.Default
List<Figure> figures = new ArrayList<>();
}

3
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/Figure.java
View File

@ -1,11 +1,10 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
import java.util.List;
import java.util.Optional;
import lombok.Builder;
@Builder
public record Figure(TextRegion caption, Region image, List<TextRegion> footnotes) {
public record Figure(Optional<TextRegion> caption, Region image) {
}

23
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/IdpResult.java
View File

@ -1,23 +0,0 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
public record IdpResult(List<KeyValuePair> keyValuePairs, List<TextRegion> handWrittenText, List<Figure> figures, List<Table> tables) {
public static IdpResult initSynchronized() {
return new IdpResult(Collections.synchronizedList(new LinkedList<>()),
Collections.synchronizedList(new LinkedList<>()),
Collections.synchronizedList(new LinkedList<>()),
Collections.synchronizedList(new LinkedList<>()));
}
public static IdpResult empty() {
return new IdpResult(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
}
}

10
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/OCRStatusUpdateResponse.java
View File

@ -1,8 +1,5 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
import java.util.Collections;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -15,16 +12,9 @@ import lombok.NoArgsConstructor;
public class OCRStatusUpdateResponse {
private String fileId;
private Set<AzureOcrFeature> features;
private int numberOfPagesToOCR;
private int numberOfOCRedPages;
private boolean ocrFinished;
private boolean ocrStarted;
public Set<AzureOcrFeature> getFeatures() {
return features == null ? Collections.emptySet() : features;
}
}

223
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/QuadPoint.java
View File

@ -5,12 +5,9 @@ import java.awt.geom.Line2D;
import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D;
import java.util.List;
import java.util.Objects;
import java.util.stream.Stream;
import lombok.Getter;
public final class QuadPoint {
public record QuadPoint(Point2D a, Point2D b, Point2D c, Point2D d) {
public enum Direction {
RIGHT,
@ -44,38 +41,7 @@ public final class QuadPoint {
* ?|_____|?
*/
}
private static final double THRESHOLD_ANGLE = Math.toRadians(5); // QuadPoint is considered straight, when its angles are below this threshold.
private final Point2D a;
private final Point2D b;
private final Point2D c;
private final Point2D d;
@Getter
private final Direction direction;
// This constructor assumes, the points form a convex polygon, I will omit the assertion for performance reasons.
public QuadPoint(Point2D a, Point2D b, Point2D c, Point2D d) {
this.a = a;
this.b = b;
this.c = c;
this.d = d;
this.direction = calculateDirection();
}
private Direction calculateDirection() {
if (isHorizontal()) {
return a.getX() < d.getX() ? Direction.RIGHT : Direction.LEFT;
}
if (isVertical()) {
return a.getY() < d.getY() ? Direction.UP : Direction.DOWN;
}
return Direction.NONE;
}
private static final double THRESHOLD_ANGLE = Math.toRadians(5); // QuadPoint is considered straight, when its angles are below this threshold
public static QuadPoint fromRectangle2D(Rectangle2D rectangle2D) {
@ -103,9 +69,7 @@ public final class QuadPoint {
public static QuadPoint fromPolygons(List<Double> polygon) {
if (polygon.size() != 8) {
throw new AssertionError();
}
assert polygon.size() == 8;
return new QuadPoint(new Point2D.Double(polygon.get(0), polygon.get(1)),
new Point2D.Double(polygon.get(6), polygon.get(7)),
new Point2D.Double(polygon.get(4), polygon.get(5)),
@ -151,6 +115,18 @@ public final class QuadPoint {
}
public Direction getDirection() {
if (isHorizontal()) {
return a.getX() < d.getX() ? Direction.RIGHT : Direction.LEFT;
}
if (isVertical()) {
return a.getY() < d.getY() ? Direction.UP : Direction.DOWN;
}
return Direction.NONE;
}
public Stream<Line2D> asLines() {
return Stream.of(new Line2D.Double(a(), b()), new Line2D.Double(b(), c()), new Line2D.Double(c(), d()), new Line2D.Double(d(), a()));
@ -170,142 +146,6 @@ public final class QuadPoint {
}
public boolean contains(double x, double y) {
// split into two triangles, test if either contains the point, assumes the QuadPoint is convex and created correctly. More specifically, the points must be in the correct order.
return triangleContains(a, b, c, x, y) || triangleContains(a, c, d, x, y);
}
/*
checks if a triangle contains a point by converting the point to barycentric coordinates using cramer's rule and then checking if the linear combination is within the bounds of the triangle.
https://en.wikipedia.org/wiki/Barycentric_coordinate_system#Barycentric_coordinates_on_triangles
*/
private boolean triangleContains(Point2D a, Point2D b, Point2D c, double x, double y) {
// area of the triangle
double denominator = ((b.getY() - c.getY()) * (a.getX() - c.getX()) + (c.getX() - b.getX()) * (a.getY() - c.getY()));
double invertedDenominator = 1.0 / denominator;
double alpha = ((b.getY() - c.getY()) * (x - c.getX()) + (c.getX() - b.getX()) * (y - c.getY())) * invertedDenominator;
double beta = ((c.getY() - a.getY()) * (x - c.getX()) + (a.getX() - c.getX()) * (y - c.getY())) * invertedDenominator;
return alpha >= 0 && beta >= 0 && alpha + beta <= 1;
}
public boolean contains(Point2D p) {
return contains(p.getX(), p.getY());
}
public boolean contains(Rectangle2D r) {
double x = r.getX();
double y = r.getY();
double maxY = r.getMaxY();
double maxX = r.getMaxX();
Point2D p1 = new Point2D.Double(x, y);
Point2D p2 = new Point2D.Double(x, maxY);
Point2D p3 = new Point2D.Double(maxX, maxY);
Point2D p4 = new Point2D.Double(maxX, y);
return contains(p1) && contains(p2) && contains(p3) && contains(p4);
}
public double getCenterX() {
return (a.getX() + b.getX() + c.getX() + d.getX()) / 4;
}
public double getCenterY() {
return (a.getY() + b.getY() + c.getY() + d.getY()) / 4;
}
public Point2D getCenter() {
return new Point2D.Double(getCenterX(), getCenterY());
}
public boolean intersects(Line2D line) {
return contains(line.getP1()) || contains(line.getP2()) || asLines().anyMatch(qLine -> qLine.intersectsLine(line));
}
public Line2D getRightLine() {
return new Line2D.Double(getTopRight(), getLowerRight());
}
public Line2D getLeftLine() {
return new Line2D.Double(getTopLeft(), getLowerLeft());
}
public Line2D getBottomLine() {
return new Line2D.Double(getLowerLeft(), getLowerRight());
}
public Line2D getTopLine() {
return new Line2D.Double(getTopLeft(), getTopRight());
}
public Point2D getTopLeft() {
return switch (direction) {
case DOWN -> a;
case LEFT -> d;
case UP -> c;
default -> b;
};
}
public Point2D getTopRight() {
return switch (direction) {
case DOWN -> b;
case LEFT -> a;
case UP -> d;
default -> c;
};
}
public Point2D getLowerRight() {
return switch (direction) {
case DOWN -> c;
case LEFT -> b;
case UP -> a;
default -> d;
};
}
public Point2D getLowerLeft() {
return switch (direction) {
case DOWN -> d;
case LEFT -> c;
case UP -> b;
default -> a;
};
}
/**
* Determines if the given QuadPoint aligns with this QuadPoint within a given threshold.
* It does os by trying every possible combination of aligning sides. It starts with the most likely combination of ab and cd.
@ -384,37 +224,4 @@ public final class QuadPoint {
return Math.atan2(deltaY, deltaX);
}
public Point2D a() {return a;}
public Point2D b() {return b;}
public Point2D c() {return c;}
public Point2D d() {return d;}
@Override
public boolean equals(Object obj) {
if (obj == this) {
return true;
}
if (obj == null || obj.getClass() != this.getClass()) {
return false;
}
var that = (QuadPoint) obj;
return Objects.equals(this.a, that.a) && Objects.equals(this.b, that.b) && Objects.equals(this.c, that.c) && Objects.equals(this.d, that.d);
}
@Override
public int hashCode() {
return Objects.hash(a, b, c, d);
}
}

5
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/QuadPointData.java
View File

@ -5,9 +5,4 @@ import lombok.Builder;
@Builder
public record QuadPointData(float[] values) {
public QuadPoint get() {
return QuadPoint.fromData(this);
}
}

7
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/Table.java
View File

@ -1,7 +0,0 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
import java.util.List;
public record Table(TextRegion caption, int numberOfCols, int numberOfRows, List<TableCell> cells, List<TextRegion> footnotes, List<Region> bboxes) {
}

5
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/TableCell.java
View File

@ -1,5 +0,0 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
public record TableCell(TextRegion textRegion, int row, int col, TableCellType kind) {
}

5
azure-ocr-service/azure-ocr-service-api/src/main/java/com/knecon/fforesight/service/ocr/v1/api/model/TableCellType.java
View File

@ -1,5 +0,0 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
public enum TableCellType {
ROW_HEADER, COLUMN_HEADER, CONTENT, STUB_HEAD, DESCRIPTION
}

2
azure-ocr-service/azure-ocr-service-processor/build.gradle.kts
View File

@ -19,7 +19,7 @@ dependencies {
implementation("com.amazonaws:aws-java-sdk-kms:1.12.440")
implementation("com.google.guava:guava:31.1-jre")
implementation("com.knecon.fforesight:viewer-doc-processor:0.193.0")
implementation("com.azure:azure-ai-documentintelligence:1.0.0")
implementation("com.azure:azure-ai-documentintelligence:1.0.0-beta.4")
implementation("com.iqser.red.commons:pdftron-logic-commons:2.32.0")

2
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/OcrServiceSettings.java
View File

@ -19,7 +19,7 @@ public class OcrServiceSettings {
boolean debug; // writes the ocr layer visibly to the viewer doc pdf
boolean drawTablesAsLines; // writes the tables to the PDF as invisible lines.
boolean snuggify = true; // attempts to shrink the word boxes returned by azure to fit the actual word pixels snug
boolean useCaches; // skips azure api, pdf rendering and image processing, when the files are already present
boolean useCaches = true; // skips azure api, pdf rendering and image processing, when the files are already present
boolean azureFontStyleDetection; // omits all image processing and uses azures FONT_STYLE feature (costs 0.6ct per page)
String contentFormat; // Either markdown or text. But, for whatever reason, with markdown enabled, key-values are not written by azure....

6
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/initializer/NativeLibrariesInitializer.java
View File

@ -21,8 +21,6 @@ public class NativeLibrariesInitializer {
@Value("${pdftron.license:}")
private String pdftronLicense;
@Value("${native-libs.path:}")
private String nativeLibsPath;
@SneakyThrows
@PostConstruct
@ -34,8 +32,8 @@ public class NativeLibrariesInitializer {
PDFNet.setTempPath("/tmp/pdftron");
PDFNet.initialize(pdftronLicense);
log.info("Setting jna.library.path: {}", nativeLibsPath);
System.setProperty("jna.library.path", nativeLibsPath);
log.info("Setting jna.library.path: {}", System.getenv("VCPKG_DYNAMIC_LIB"));
System.setProperty("jna.library.path", System.getenv("VCPKG_DYNAMIC_LIB"));
log.info("Asserting Native Libraries loaded");

102
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/DocumentSpanLookup.java
View File

@ -1,102 +0,0 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
import com.azure.ai.documentintelligence.models.AnalyzeResult;
import com.azure.ai.documentintelligence.models.DocumentPage;
import com.azure.ai.documentintelligence.models.DocumentSpan;
import com.azure.ai.documentintelligence.models.DocumentWord;
public class DocumentSpanLookup {
List<PageSpanLookup> documentWordLookup;
public DocumentSpanLookup(AnalyzeResult analyzeResult) {
documentWordLookup = new ArrayList<>(analyzeResult.getPages().size());
int offset = 0;
for (DocumentPage page : analyzeResult.getPages()) {
if (page.getWords() == null || page.getWords().isEmpty()) {
documentWordLookup.add(new PageSpanLookup(offset, offset, null));
}
int start = page.getWords()
.get(0).getSpan().getOffset();
DocumentSpan span = page.getWords()
.get(page.getWords().size() - 1).getSpan();
int end = span.getOffset() + span.getLength();
SpanLookup<DocumentWord> pageWords = new SpanLookup<>(page.getWords()
.stream(), DocumentWord::getSpan);
documentWordLookup.add(new PageSpanLookup(start, end, pageWords));
offset = end + 1;
}
}
public List<WordOnPage> findWordsOnPages(DocumentSpan documentSpan) {
if (documentSpan == null) {
return Collections.emptyList();
}
int firstSmallerIdx = findIdxOfFirstSmallerObject(documentSpan);
PageSpanLookup firstPage = documentWordLookup.get(firstSmallerIdx);
List<WordOnPage> wordsOnPages = new ArrayList<>();
for (int pageNumber = firstSmallerIdx; pageNumber < documentWordLookup.size(); pageNumber++) {
PageSpanLookup page = documentWordLookup.get(pageNumber);
if (page.end >= documentSpan.getOffset()) {
break;
}
firstPage.wordSpanLookup.findElementsContainedInSpan(documentSpan)
.stream()
.map(documentWord -> new WordOnPage(documentWord, firstSmallerIdx))
.forEach(wordsOnPages::add);
}
return wordsOnPages;
}
private int findIdxOfFirstSmallerObject(DocumentSpan documentSpan) {
int idx = Collections.binarySearch(documentWordLookup, new PageSpanLookup(documentSpan.getOffset(), -1, null), Comparator.comparing(PageSpanLookup::start));
if (idx >= 0) {
return idx;
} else {
int insertionPoint = -(idx + 1);
if (insertionPoint == 0) {
return -1;
}
var lastSmaller = documentWordLookup.get(insertionPoint - 1);
for (int resultIdx = insertionPoint - 2; resultIdx >= 0; resultIdx--) {
if (documentWordLookup.get(resultIdx).compareTo(lastSmaller) == 0) {
return resultIdx + 1;
}
}
return 0;
}
}
public record WordOnPage(DocumentWord documentWord, int pageNumber) {
}
private record PageSpanLookup(int start, int end, SpanLookup<DocumentWord> wordSpanLookup) implements Comparable<PageSpanLookup> {
@Override
public int compareTo(PageSpanLookup o) {
return Integer.compare(start, o.start);
}
}
}

5
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/model/PageInformation.java
View File

@ -13,7 +13,7 @@ import com.pdftron.pdf.Rect;
import lombok.SneakyThrows;
public record PageInformation(Rectangle2D mediabox, Rectangle2D cropBox, int number, int rotationDegrees, List<Rectangle2D> wordBBoxes) {
public record PageInformation(Rectangle2D mediabox, int number, int rotationDegrees, List<Rectangle2D> wordBBoxes) {
@SneakyThrows
public static Map<Integer, PageInformation> fromPDFDoc(PDFDoc pdfDoc) {
@ -34,9 +34,8 @@ public record PageInformation(Rectangle2D mediabox, Rectangle2D cropBox, int num
@SneakyThrows
public static PageInformation fromPage(int pageNum, Page page) {
try (Rect mediaBox = page.getCropBox(); Rect cropBox = page.getCropBox()) {
try (Rect mediaBox = page.getCropBox()) {
return new PageInformation(new Rectangle2D.Double(mediaBox.getX1(), mediaBox.getY1(), mediaBox.getWidth(), mediaBox.getHeight()),
new Rectangle2D.Double(cropBox.getX1(), cropBox.getY1(), cropBox.getWidth(), cropBox.getHeight()),
pageNum,
page.getRotation() * 90,
DocumentTextExtractor.getTextBBoxes(page));

2
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/AsyncOcrService.java
View File

@ -123,7 +123,7 @@ public class AsyncOcrService {
private static void handleCompleted(BatchContext batchContext) {
log.info("Batch {}: Completed with pages {}", batchContext.batch.getIndex(), batchContext.batch);
log.info("Completed batch {} with pages {}", batchContext.batch.getIndex(), batchContext.batch);
}

32
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/AzureOcrResource.java
View File

@ -1,19 +1,21 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.yaml.snakeyaml.events.Event;
import com.azure.ai.documentintelligence.DocumentIntelligenceAsyncClient;
import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder;
import com.azure.ai.documentintelligence.models.AnalyzeDocumentOptions;
import com.azure.ai.documentintelligence.models.AnalyzeOperationDetails;
import com.azure.ai.documentintelligence.models.AnalyzeDocumentRequest;
import com.azure.ai.documentintelligence.models.AnalyzeResult;
import com.azure.ai.documentintelligence.models.AnalyzeResultOperation;
import com.azure.ai.documentintelligence.models.ContentFormat;
import com.azure.ai.documentintelligence.models.DocumentAnalysisFeature;
import com.azure.ai.documentintelligence.models.DocumentContentFormat;
import com.azure.ai.documentintelligence.models.StringIndexType;
import com.azure.core.credential.AzureKeyCredential;
import com.azure.core.util.BinaryData;
@ -44,23 +46,29 @@ public class AzureOcrResource {
@SneakyThrows
public PollerFlux<AnalyzeOperationDetails, AnalyzeResult> callAzureAsync(BinaryData data, Set<AzureOcrFeature> features) {
public PollerFlux<AnalyzeResultOperation, AnalyzeResult> callAzureAsync(BinaryData data, Set<AzureOcrFeature> features) {
AnalyzeDocumentOptions analyzeDocumentOptions = new AnalyzeDocumentOptions(data.toBytes());
analyzeDocumentOptions.setStringIndexType(StringIndexType.UTF16_CODE_UNIT);
analyzeDocumentOptions.setDocumentAnalysisFeatures(buildFeatures(features));
analyzeDocumentOptions.setOutputContentFormat(buildContentFormat());
return asyncClient.beginAnalyzeDocument(getModelId(features), analyzeDocumentOptions);
AnalyzeDocumentRequest analyzeRequest = new AnalyzeDocumentRequest().setBase64Source(data.toBytes());
return asyncClient.beginAnalyzeDocument(getModelId(features),
null,
null,
StringIndexType.UTF16CODE_UNIT,
buildFeatures(features),
null,
buildContentFormat(),
Collections.emptyList(),
analyzeRequest);
}
private DocumentContentFormat buildContentFormat() {
private ContentFormat buildContentFormat() {
if (Objects.equal(settings.getContentFormat(), "markdown")) {
return DocumentContentFormat.MARKDOWN;
return ContentFormat.MARKDOWN;
}
return DocumentContentFormat.TEXT;
return ContentFormat.TEXT;
}

16
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/IOcrMessageSender.java
View File

@ -1,24 +1,16 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.util.Set;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
@Service
public interface IOcrMessageSender {
void sendUpdate(String fileId, int finishedImages, int totalImages, Set<AzureOcrFeature> features);
void sendUpdate(String fileId, int finishedImages, int totalImages);
void sendOCRStarted(String fileId);
void sendOCRStarted(String fileId, Set<AzureOcrFeature> features);
void sendOcrFinished(String fileId, int totalImages);
void sendOcrFinished(String fileId, int totalImages, Set<AzureOcrFeature> features);
void sendOcrResponse(DocumentRequest request);
void sendOcrResponse(String dossierId, String fileId);
}

26
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OCRService.java
View File

@ -5,7 +5,6 @@ import static com.knecon.fforesight.service.ocr.processor.model.Statistics.human
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
@ -14,7 +13,6 @@ import java.util.Set;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
import com.iqser.red.pdftronlogic.commons.OCGWatermarkRemovalService;
import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
@ -47,7 +45,6 @@ public class OCRService {
BatchFactory batchFactory;
AsyncOcrService asyncOcrService;
OcrServiceSettings settings;
ObjectMapper mapper;
/**
@ -60,10 +57,10 @@ public class OCRService {
* @param tmpDir working directory for all files
* @param documentFile the file to perform ocr on, results are written invisibly
* @param viewerDocumentFile debugging file, results are written visibly in an optional content group
* @param idpResultFile result file with additional information
* @param analyzeResultFile result file with additional information
*/
@Observed(name = "OCRService", contextualName = "run-ocr-on-document")
public void runOcrOnDocument(String dossierId, String fileId, Set<AzureOcrFeature> features, Path tmpDir, File documentFile, File viewerDocumentFile, File idpResultFile) {
public void runOcrOnDocument(String dossierId, String fileId, Set<AzureOcrFeature> features, Path tmpDir, File documentFile, File viewerDocumentFile, File analyzeResultFile) {
if (features.contains(AzureOcrFeature.REMOVE_WATERMARKS)) {
removeWatermark(documentFile);
@ -71,9 +68,10 @@ public class OCRService {
removeInvisibleElements(documentFile);
log.info("Starting OCR");
long ocrStart = System.currentTimeMillis();
Statistics stats = runOcr(tmpDir, documentFile, viewerDocumentFile, fileId, dossierId, idpResultFile, features).getStatistics();
Statistics stats = runOcr(tmpDir, documentFile, viewerDocumentFile, fileId, dossierId, analyzeResultFile, features).getStatistics();
long ocrEnd = System.currentTimeMillis();
log.info("OCR successful, took {}", humanizeDuration(ocrEnd - ocrStart));
@ -125,14 +123,14 @@ public class OCRService {
File viewerDocumentFile,
String fileId,
String dossierId,
File idpResultFile,
File analyzeResultFile,
Set<AzureOcrFeature> features) {
try (var in = new FileInputStream(documentFile); PDFDoc pdfDoc = new PDFDoc(in)) {
OCGWatermarkRemovalService.removeWatermarks(pdfDoc);
OcrExecutionSupervisor supervisor = new OcrExecutionSupervisor(pdfDoc.getPageCount(), ocrMessageSender, fileId, settings, features);
OcrExecutionSupervisor supervisor = new OcrExecutionSupervisor(pdfDoc.getPageCount(), ocrMessageSender, fileId, settings);
supervisor.getStatistics().setStart();
List<PageBatch> batches = batchFactory.splitIntoBatches(pdfDoc, supervisor, features, runDir);
@ -147,10 +145,6 @@ public class OCRService {
RotationCorrectionUtility.rotatePages(viewerDocumentFile.toPath(), viewerDocumentFile.toPath(), ocrResult.anglesPerPage());
}
if (features.contains(AzureOcrFeature.IDP)) {
saveIdpResultFile(idpResultFile, ocrResult);
}
supervisor.getStatistics().drawingPdfFinished();
supervisor.sendFinished();
@ -160,12 +154,4 @@ public class OCRService {
}
private void saveIdpResultFile(File idpResultFile, OcrResult ocrResult) throws IOException {
try (var out = new FileOutputStream(idpResultFile)) {
mapper.writeValue(out, ocrResult.idpResult());
}
}
}

17
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OcrExecutionSupervisor.java
View File

@ -14,7 +14,6 @@ import java.util.concurrent.CountDownLatch;
import com.knecon.fforesight.service.ocr.processor.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.model.PageBatch;
import com.knecon.fforesight.service.ocr.processor.model.Statistics;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
import lombok.AccessLevel;
import lombok.Getter;
@ -40,15 +39,12 @@ public class OcrExecutionSupervisor {
String fileId;
Set<AzureOcrFeature> features;
public OcrExecutionSupervisor(int totalPageCount, IOcrMessageSender ocrMessageSender, String fileId, OcrServiceSettings settings, Set<AzureOcrFeature> features) {
public OcrExecutionSupervisor(int totalPageCount, IOcrMessageSender ocrMessageSender, String fileId, OcrServiceSettings settings) {
this.totalPageCount = totalPageCount;
this.ocrMessageSender = ocrMessageSender;
this.fileId = fileId;
this.features = features;
this.errorPages = Collections.synchronizedSet(new HashSet<>());
this.countDownPagesToProcess = new CountDownLatch(totalPageCount);
this.statistics = new Statistics();
@ -90,7 +86,7 @@ public class OcrExecutionSupervisor {
if (!statistics.getBatchStats(pageRange).isUploadFinished()) {
log.info("Batch {}: Pages {} is in progress", pageRange.getIndex(), pageRange);
statistics.getBatchStats(pageRange).finishUpload();
ocrMessageSender.sendUpdate(fileId, processedPages(), getTotalPageCount(), features);
ocrMessageSender.sendUpdate(fileId, processedPages(), getTotalPageCount());
} else {
log.debug("Batch {}: Pages {} still in progress", pageRange.getIndex(), pageRange);
}
@ -101,15 +97,14 @@ public class OcrExecutionSupervisor {
batch.forEach(pageIndex -> countDownPagesToProcess.countDown());
statistics.getBatchStats(batch).finishMappingResult();
ocrMessageSender.sendUpdate(fileId, this.processedPages(), getTotalPageCount(), features);
log.info("Batch {}: Finished mapping result with pages {}", batch.getIndex(), batch);
ocrMessageSender.sendUpdate(fileId, this.processedPages(), getTotalPageCount());
}
public void logPageSkipped(Integer pageIndex) {
this.countDownPagesToProcess.countDown();
ocrMessageSender.sendUpdate(fileId, this.processedPages(), getTotalPageCount(), features);
ocrMessageSender.sendUpdate(fileId, this.processedPages(), getTotalPageCount());
log.debug("{}/{}: No images to ocr on page {}", processedPages(), getTotalPageCount(), pageIndex);
}
@ -119,7 +114,7 @@ public class OcrExecutionSupervisor {
this.errorPages.add(batch);
batch.forEach(pageIndex -> this.countDownPagesToProcess.countDown());
ocrMessageSender.sendUpdate(fileId, this.processedPages(), getTotalPageCount(), features);
ocrMessageSender.sendUpdate(fileId, this.processedPages(), getTotalPageCount());
log.error("{}/{}: Error occurred in batch {} with pages {}", processedPages(), getTotalPageCount(), batch.getIndex(), batch, e);
}
@ -172,7 +167,7 @@ public class OcrExecutionSupervisor {
requireNoErrors();
log.info("{}/{}: Finished OCR on all pages", getTotalPageCount(), getTotalPageCount());
ocrMessageSender.sendOcrFinished(fileId, getTotalPageCount(), features);
ocrMessageSender.sendOcrFinished(fileId, getTotalPageCount());
}

14
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/OcrResultPostProcessingPipeline.java
View File

@ -18,13 +18,13 @@ import java.util.stream.Stream;
import com.azure.ai.documentintelligence.models.AnalyzeResult;
import com.azure.ai.documentintelligence.models.BoundingRegion;
import com.azure.ai.documentintelligence.models.DocumentFontStyle;
import com.azure.ai.documentintelligence.models.DocumentPage;
import com.azure.ai.documentintelligence.models.DocumentSpan;
import com.azure.ai.documentintelligence.models.DocumentStyle;
import com.azure.ai.documentintelligence.models.DocumentTable;
import com.azure.ai.documentintelligence.models.DocumentTableCell;
import com.azure.ai.documentintelligence.models.DocumentWord;
import com.azure.ai.documentintelligence.models.FontWeight;
import com.google.common.base.Functions;
import com.knecon.fforesight.service.ocr.processor.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.model.ImageFile;
@ -32,10 +32,10 @@ import com.knecon.fforesight.service.ocr.processor.model.PageBatch;
import com.knecon.fforesight.service.ocr.processor.model.PageInformation;
import com.knecon.fforesight.service.ocr.processor.model.SpanLookup;
import com.knecon.fforesight.service.ocr.processor.model.TextPositionInImage;
import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.BBoxSnuggificationService;
import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.FontStyleDetector;
import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.ImageProcessingPipeline;
import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.ImageProcessingSupervisor;
import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.BBoxSnuggificationService;
import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.StrokeWidthCalculator;
import com.knecon.fforesight.service.ocr.processor.visualizations.WritableOcrResult;
import com.knecon.fforesight.service.ocr.processor.visualizations.fonts.FontMetricsProvider;
@ -106,7 +106,6 @@ public class OcrResultPostProcessingPipeline {
writableOcrResultList.add(builder.build());
}
log.debug("Batch {}: finished post-processing.", batch.getIndex());
return writableOcrResultList;
}
@ -323,13 +322,16 @@ public class OcrResultPostProcessingPipeline {
return Lookups.empty();
}
// Azure stopped supporting bold text detection in 1.0.0 release
SpanLookup<DocumentSpan> boldLookup = new SpanLookup<>(Stream.empty(), Function.identity());
SpanLookup<DocumentSpan> boldLookup = new SpanLookup<>(analyzeResult.getStyles()
.stream()
.filter(style -> Objects.equals(style.getFontWeight(), FontWeight.BOLD))
.map(DocumentStyle::getSpans)
.flatMap(Collection::stream), Function.identity());
SpanLookup<DocumentSpan> italicLookup = new SpanLookup<>(analyzeResult.getStyles()
.stream()
.filter(style -> Objects.equals(style.getFontStyle(),
DocumentFontStyle.ITALIC))
com.azure.ai.documentintelligence.models.FontStyle.ITALIC))
.map(DocumentStyle::getSpans)
.flatMap(Collection::stream), Functions.identity());

11
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/BBoxSnuggificationService.java
View File

@ -31,7 +31,7 @@ public class BBoxSnuggificationService {
private static final double AVERAGE_ANGLE_THRESHOLD = 0.2; // Skips snuggification, if the average remaining word rotation of a word, written from left-to-right is bigger than this
public static final int INDIVIDUAL_ANGLE_THRESHOLD = 5; // skips snuggification for word, if the remaining rotation is larger than this angle
public static final int MAX_SHRINK_PIXELS = 40; // Number of pixels that are allowed to be removed from the top or bottom of an image
private static final int MINIMUM_WORD_PIXELS = 5; // Number of pixels that are required for snuggification
private static final int MINIMUM_WORD_Pixels = 5;
private enum Operation {
HORIZONTAL,
@ -48,11 +48,6 @@ public class BBoxSnuggificationService {
return Optional.empty();
}
if (origin.getContent().equals("-") || origin.getContent().equals(",")) {
// very slim characters should not be snuggified, or the fontsize may be off significantly
return Optional.empty();
}
QuadPoint originTransformed = QuadPoint.fromPolygons(origin.getPolygon()).getTransformed(resultToImageTransform);
double remainingAngle = Math.abs(RotationCorrectionUtility.getRemainingAngle(originTransformed.getAngle()));
QuadPoint.Direction direction = originTransformed.getDirection();
@ -138,7 +133,7 @@ public class BBoxSnuggificationService {
if (start == 0 && end == wordImage.w) {
return Optional.empty();
}
if (Math.abs(start - end) < MINIMUM_WORD_PIXELS) {
if (Math.abs(start - end) < MINIMUM_WORD_Pixels) {
return Optional.empty();
}
return Optional.of(new Rectangle2D.Double(origin.getX() + start, origin.getY(), origin.getWidth() - start - (wordImage.w - end), origin.getHeight()));
@ -164,7 +159,7 @@ public class BBoxSnuggificationService {
if (start == 0 && end == wordImage.h) {
return Optional.empty();
}
if (Math.abs(start - end) < MINIMUM_WORD_PIXELS) {
if (Math.abs(start - end) < MINIMUM_WORD_Pixels) {
return Optional.empty();
}
return Optional.of(new Rectangle2D.Double(origin.getX(), origin.getY() + start, origin.getWidth(), origin.getHeight() - start - (wordImage.h - end)));

16
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/GhostScriptOutputHandler.java
View File

@ -15,6 +15,7 @@ import java.util.regex.Pattern;
import org.slf4j.MDC;
import com.knecon.fforesight.service.ocr.processor.model.ImageFile;
import com.knecon.fforesight.service.ocr.processor.model.PageBatch;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -75,14 +76,13 @@ public class GhostScriptOutputHandler extends Thread {
if (line == null) {
break;
}
switch (type) {
case STD_OUT -> {
log.debug("Batch {}: {}_{}>{}", batchIdx, processName, type.name(), line);
addProcessedImageToQueue(line);
}
case ERROR -> log.error("Batch {}: {}_{}>{}", batchIdx, processName, type.name(), line);
}
if (type.equals(Type.ERROR)) {
log.error("{}_{}>{}", processName, type.name(), line);
} else {
log.debug("{}_{}>{}", processName, type.name(), line);
addProcessedImageToQueue(line);
}
}
}
is.close();
@ -92,7 +92,7 @@ public class GhostScriptOutputHandler extends Thread {
if (!pagesToProcess.isEmpty()) {
errorHandler.accept(String.format("Ghostscript finished for batch %d, but pages %s remain unprocessed.", batchIdx, formatPagesToProcess()));
} else {
log.info("Batch {}: rendered successfully!", batchIdx);
log.info("Batch: {} rendered successfully!", batchIdx);
}
}

50
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/GhostScriptService.java
View File

@ -6,7 +6,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import org.springframework.stereotype.Service;
@ -16,12 +15,14 @@ import com.knecon.fforesight.service.ocr.processor.model.ImageFile;
import com.knecon.fforesight.service.ocr.processor.model.PageBatch;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@SuppressWarnings("PMD") // can't figure out how to safely close the stdOut and stdError streams in line 72/74
public class GhostScriptService {
@ -33,37 +34,6 @@ public class GhostScriptService {
private Semaphore concurrencySemaphore = new Semaphore(3);
public GhostScriptService(OcrServiceSettings ocrServiceSettings) {
this.ocrServiceSettings = ocrServiceSettings;
assertGhostscriptIsInstalled();
}
private void assertGhostscriptIsInstalled() {
try {
Process p = Runtime.getRuntime().exec("gs -v");
InputStream stdOut = p.getInputStream();
InputStream errOut = p.getErrorStream();
assert p.waitFor(1, TimeUnit.SECONDS);
log.info("Ghostscript is installed.");
String out = new String(stdOut.readAllBytes());
String error = new String(errOut.readAllBytes());
for (String line : out.split("\n")) {
log.info(line);
}
if (!error.isBlank()) {
log.error(error);
}
} catch (Exception e) {
log.error("Ghostscript is not installed!");
log.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
@SneakyThrows
public void startBatchRender(PageBatch batch, ImageProcessingSupervisor supervisor, Consumer<ImageFile> successHandler, Consumer<String> errorHandler) {
@ -79,7 +49,7 @@ public class GhostScriptService {
concurrencySemaphore.acquire();
log.info("Batch {}: starting GhostScript rendering with page(s) {}", batch.getIndex(), batch);
executeProcess(batch, buildCmdArgs(batch, batch.getBatchDoc()), successHandler, errorHandler);
executeProcess(batch.getIndex(), buildCmdArgs(batch, batch.getBatchDoc()), successHandler, errorHandler);
}
@ -106,27 +76,27 @@ public class GhostScriptService {
@SneakyThrows
private void executeProcess(PageBatch batch, ProcessCmdsAndRenderedImageFiles processInfo, Consumer<ImageFile> successHandler, Consumer<String> errorHandler) {
private void executeProcess(int batchIdx, ProcessCmdsAndRenderedImageFiles processInfo, Consumer<ImageFile> successHandler, Consumer<String> errorHandler) {
Process p = Runtime.getRuntime().exec(processInfo.cmdArgs());
InputStream stdOut = p.getInputStream();
GhostScriptOutputHandler stdOutLogger = GhostScriptOutputHandler.stdOut(batch.getIndex(), stdOut, processInfo.renderedPageImageFiles(), successHandler, errorHandler);
GhostScriptOutputHandler stdOutLogger = GhostScriptOutputHandler.stdOut(batchIdx, stdOut, processInfo.renderedPageImageFiles(), successHandler, errorHandler);
InputStream stdError = p.getErrorStream();
GhostScriptOutputHandler stdErrorLogger = GhostScriptOutputHandler.stdError(batch.getIndex(), stdError, errorHandler);
GhostScriptOutputHandler stdErrorLogger = GhostScriptOutputHandler.stdError(batchIdx, stdError, errorHandler);
stdOutLogger.start();
stdErrorLogger.start();
handleFinished(p, errorHandler, batch, successHandler);
handleFinished(p);
}
private void handleFinished(Process p, Consumer<String> errorHandler, PageBatch batch, Consumer<ImageFile> successHandler) {
private void handleFinished(Process p) {
Thread finishedThread = new Thread(() -> {
try {
p.waitFor(2, TimeUnit.MINUTES);
p.waitFor();
} catch (InterruptedException e) {
errorHandler.accept("Batch %d: Ghostscript rendering has been terminated after 2 minutes \n %s".formatted(batch.getIndex(), e.getMessage()));
log.error("GhostScript process was interrupted", e);
} finally {
concurrencySemaphore.release();
}

3
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/ImageProcessingService.java
View File

@ -87,10 +87,9 @@ public class ImageProcessingService {
LeptUtils.disposePix(processedPix);
}
} catch (Exception e) {
supervisor.markError("Page %d could not be processed due to: %s".formatted(unprocessedImage.pageNumber(), e.getMessage()));
supervisor.markError(e.getMessage());
} finally {
supervisor.markPageFinished(processedImage);
log.debug("Finished page: {}", processedImage.pageNumber());
}
}

1
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/service/imageprocessing/ImageProcessingSupervisor.java
View File

@ -69,7 +69,6 @@ public class ImageProcessingSupervisor {
public void markError(String errorMessage) {
log.error(errorMessage);
this.errors.add(errorMessage);
}

40
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/utils/StringCleaningUtility.java
View File

@ -1,40 +0,0 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import java.util.regex.Pattern;
import lombok.experimental.UtilityClass;
@UtilityClass
public class StringCleaningUtility {
public static final Pattern hyphenLineBreaks = Pattern.compile("[-~‐‒⁻−﹣゠⁓‑\\u00AD][\\r\\n]+");
public static final Pattern linebreaks = Pattern.compile("[\\r\\n]+");
public static final Pattern doubleWhitespaces = Pattern.compile("\\s{2,}");
public static String cleanString(String value) {
String noHyphenLinebreaks = removeHyphenLinebreaks(value);
String noLinebreaks = removeLinebreaks(noHyphenLinebreaks);
return removeMultipleWhitespaces(noLinebreaks);
}
private String removeHyphenLinebreaks(String value) {
return hyphenLineBreaks.matcher(value).replaceAll("");
}
private String removeMultipleWhitespaces(String value) {
return doubleWhitespaces.matcher(value).replaceAll(" ");
}
private String removeLinebreaks(String value) {
return linebreaks.matcher(value).replaceAll(" ");
}
}

4
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/AnalyzeResultMapper.java
View File

@ -1,14 +1,14 @@
package com.knecon.fforesight.service.ocr.processor.visualizations;
import com.azure.ai.documentintelligence.models.AnalyzeResult;
import com.knecon.fforesight.service.ocr.v1.api.model.IdpResult;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureAnalyzeResult;
import lombok.experimental.UtilityClass;
@UtilityClass
public class AnalyzeResultMapper {
public IdpResult map(AnalyzeResult analyzeResult) {
public AzureAnalyzeResult map(AnalyzeResult analyzeResult) {
return null;
}

17
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/layers/IdpLayer.java
View File

@ -12,6 +12,8 @@ import com.azure.ai.documentintelligence.models.DocumentBarcode;
import com.azure.ai.documentintelligence.models.DocumentFigure;
import com.azure.ai.documentintelligence.models.DocumentKeyValuePair;
import com.azure.ai.documentintelligence.models.DocumentLine;
import com.azure.ai.documentintelligence.models.DocumentList;
import com.azure.ai.documentintelligence.models.DocumentListItem;
import com.azure.ai.documentintelligence.models.DocumentParagraph;
import com.azure.ai.documentintelligence.models.DocumentSection;
import com.azure.ai.documentintelligence.models.DocumentTable;
@ -21,8 +23,8 @@ import com.azure.ai.documentintelligence.models.DocumentWord;
import com.azure.ai.documentintelligence.models.ParagraphRole;
import com.knecon.fforesight.service.ocr.processor.model.PageBatch;
import com.knecon.fforesight.service.ocr.processor.model.SpanLookup;
import com.knecon.fforesight.service.ocr.processor.visualizations.utils.LineUtils;
import com.knecon.fforesight.service.ocr.processor.visualizations.utils.Rectangle2DBBoxCollector;
import com.knecon.fforesight.service.ocr.processor.visualizations.utils.LineUtils;
import com.knecon.fforesight.service.ocr.v1.api.model.QuadPoint;
import com.knecon.fforesight.service.viewerdoc.layers.IdpLayerConfig;
import com.knecon.fforesight.service.viewerdoc.model.ColoredLine;
@ -67,6 +69,14 @@ public class IdpLayer extends IdpLayerConfig {
}
public void addList(DocumentList list, PageBatch pageOffset) {
for (DocumentListItem item : list.getItems()) {
addBoundingRegion(item.getBoundingRegions(), lists, PARAGRAPH_COLOR, pageOffset);
}
}
public void addBarcode(int pageNumber, DocumentBarcode barcode) {
addPolygon(pageNumber, barcode.getPolygon(), barcodes, IMAGE_COLOR);
@ -75,11 +85,8 @@ public class IdpLayer extends IdpLayerConfig {
public void addKeyValue(DocumentKeyValuePair keyValue, PageBatch pageOffset) {
if (keyValue.getKey() == null || keyValue.getKey().getContent().isEmpty()) {
return;
}
addBoundingRegion(keyValue.getKey().getBoundingRegions(), keyValuePairs, KEY_COLOR, pageOffset);
if (keyValue.getValue() != null && !keyValue.getValue().getContent().isEmpty()) {
if (keyValue.getValue() != null) {
addBoundingRegion(keyValue.getValue().getBoundingRegions(), keyValuePairs, VALUE_COLOR, pageOffset);
if (keyValue.getKey().getBoundingRegions()

241
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/layers/IdpResultFactory.java
View File

@ -1,241 +0,0 @@
package com.knecon.fforesight.service.ocr.processor.visualizations.layers;
import static com.knecon.fforesight.service.ocr.processor.utils.StringCleaningUtility.cleanString;
import java.awt.geom.AffineTransform;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.azure.ai.documentintelligence.models.AnalyzeResult;
import com.azure.ai.documentintelligence.models.BoundingRegion;
import com.azure.ai.documentintelligence.models.DocumentCaption;
import com.azure.ai.documentintelligence.models.DocumentFigure;
import com.azure.ai.documentintelligence.models.DocumentFootnote;
import com.azure.ai.documentintelligence.models.DocumentKeyValuePair;
import com.azure.ai.documentintelligence.models.DocumentTable;
import com.azure.ai.documentintelligence.models.DocumentTableCell;
import com.knecon.fforesight.service.ocr.processor.model.DocumentSpanLookup;
import com.knecon.fforesight.service.ocr.processor.model.PageBatch;
import com.knecon.fforesight.service.ocr.processor.model.PageInformation;
import com.knecon.fforesight.service.ocr.processor.visualizations.utils.Rectangle2DBBoxCollector;
import com.knecon.fforesight.service.ocr.processor.visualizations.utils.RotationCorrectionUtility;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
import com.knecon.fforesight.service.ocr.v1.api.model.Figure;
import com.knecon.fforesight.service.ocr.v1.api.model.IdpResult;
import com.knecon.fforesight.service.ocr.v1.api.model.KeyValuePair;
import com.knecon.fforesight.service.ocr.v1.api.model.QuadPoint;
import com.knecon.fforesight.service.ocr.v1.api.model.Region;
import com.knecon.fforesight.service.ocr.v1.api.model.Table;
import com.knecon.fforesight.service.ocr.v1.api.model.TableCell;
import com.knecon.fforesight.service.ocr.v1.api.model.TableCellType;
import com.knecon.fforesight.service.ocr.v1.api.model.TextRegion;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Getter
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class IdpResultFactory {
IdpResult idpResult;
Map<Integer, AffineTransform> resultToPageTransforms;
Map<Integer, PageInformation> pageInformation;
Map<Integer, Double> angles;
boolean rotationCorrection;
public IdpResultFactory(Map<Integer, AffineTransform> resultToPageTransforms,
Map<Integer, PageInformation> pageInformation,
Map<Integer, Double> angles,
Set<AzureOcrFeature> features) {
this.angles = angles;
this.rotationCorrection = features.contains(AzureOcrFeature.ROTATION_CORRECTION);
this.resultToPageTransforms = resultToPageTransforms;
this.pageInformation = pageInformation;
this.idpResult = IdpResult.initSynchronized();
}
public AffineTransform getResultToPageTransform(Integer pageNumber) {
AffineTransform transform;
if (rotationCorrection) {
PageInformation page = pageInformation.get(pageNumber);
transform = RotationCorrectionUtility.buildTransform(-angles.get(pageNumber), page.cropBox().getWidth(), page.cropBox().getHeight(), false);
} else {
transform = new AffineTransform();
}
transform.concatenate(resultToPageTransforms.get(pageNumber));
return transform;
}
public void addAnalyzeResult(AnalyzeResult analyzeResult, PageBatch batch) {
DocumentSpanLookup words = new DocumentSpanLookup(analyzeResult);
if (analyzeResult.getTables() != null) {
analyzeResult.getTables()
.forEach(documentTable -> addTable(documentTable, words, batch));
}
if (analyzeResult.getKeyValuePairs() != null) {
analyzeResult.getKeyValuePairs()
.forEach(documentKeyValuePair -> addKeyValuePair(documentKeyValuePair, batch));
}
if (analyzeResult.getFigures() != null) {
analyzeResult.getFigures()
.forEach(documentFigure -> addFigure(documentFigure, batch, words));
}
}
private void addFigure(DocumentFigure documentFigure, PageBatch batch, DocumentSpanLookup words) {
List<TextRegion> footNotes = new LinkedList<>();
if (documentFigure.getFootnotes() != null) {
documentFigure.getFootnotes()
.stream()
.map(footNote -> toTextRegion(footNote, batch))
.filter(Objects::nonNull)
.forEach(footNotes::add);
}
int batchPageNumber = documentFigure.getBoundingRegions()
.get(0).getPageNumber();
Region bbox = toRegionFromRegions(batch.getPageNumber(batchPageNumber), documentFigure.getBoundingRegions());
TextRegion caption = toTextRegion(documentFigure.getCaption(), batch);
idpResult.figures().add(new Figure(caption, bbox, footNotes));
}
private void addKeyValuePair(DocumentKeyValuePair documentKeyValuePair, PageBatch batch) {
TextRegion key = null;
if (documentKeyValuePair.getKey() != null && !documentKeyValuePair.getKey().getContent().isEmpty()) {
Region region = toRegionFromRegions(batch, documentKeyValuePair.getKey().getBoundingRegions());
key = new TextRegion(region, cleanString(documentKeyValuePair.getKey().getContent()));
}
TextRegion value = null;
if (documentKeyValuePair.getValue() != null && !documentKeyValuePair.getValue().getContent().isEmpty()) {
Region region = toRegionFromRegions(batch, documentKeyValuePair.getValue().getBoundingRegions());
value = new TextRegion(region, cleanString(documentKeyValuePair.getValue().getContent()));
}
idpResult.keyValuePairs().add(new KeyValuePair(key, value));
}
private void addTable(DocumentTable documentTable, DocumentSpanLookup words, PageBatch batch) {
TextRegion caption = toTextRegion(documentTable.getCaption(), batch);
List<TableCell> tableCells = documentTable.getCells()
.stream()
.map(documentTableCell -> toTableCell(documentTableCell, words, batch))
.toList();
List<TextRegion> footNotes = new LinkedList<>();
if (documentTable.getFootnotes() != null) {
documentTable.getFootnotes()
.stream()
.map(footNote -> toTextRegion(footNote, batch))
.filter(Objects::nonNull)
.forEach(footNotes::add);
}
List<Region> bbox = documentTable.getBoundingRegions()
.stream()
.map(b -> toRegion(b, batch))
.toList();
Table table = new Table(caption, documentTable.getColumnCount(), documentTable.getRowCount(), tableCells, footNotes, bbox);
idpResult.tables().add(table);
}
private TextRegion toTextRegion(DocumentFootnote footNote, PageBatch batch) {
if (footNote == null || footNote.getBoundingRegions().isEmpty()) {
return null;
}
Region region = toRegionFromRegions(batch, footNote.getBoundingRegions());
return new TextRegion(region, cleanString(footNote.getContent()));
}
private TextRegion toTextRegion(DocumentCaption caption, PageBatch batch) {
if (caption == null || caption.getBoundingRegions().isEmpty()) {
return null;
}
Region region = toRegionFromRegions(batch, caption.getBoundingRegions());
return new TextRegion(region, cleanString(caption.getContent()));
}
private TableCell toTableCell(DocumentTableCell documentTableCell, DocumentSpanLookup words, PageBatch batch) {
int batchPageNumber = documentTableCell.getBoundingRegions()
.get(0).getPageNumber();
Region region = toRegionFromRegions(batch.getPageNumber(batchPageNumber), documentTableCell.getBoundingRegions());
TableCellType kind = mapTableCellType(documentTableCell);
return new TableCell(new TextRegion(region, cleanString(documentTableCell.getContent())), documentTableCell.getRowIndex(), documentTableCell.getColumnIndex(), kind);
}
private static TableCellType mapTableCellType(DocumentTableCell documentTableCell) {
if (documentTableCell.getKind() == null) {
return TableCellType.CONTENT;
}
return switch (documentTableCell.getKind().toString()) {
case "columnHeader" -> TableCellType.COLUMN_HEADER;
case "rowHeader" -> TableCellType.ROW_HEADER;
case "description" -> TableCellType.DESCRIPTION;
case "stubHead" -> TableCellType.STUB_HEAD;
default -> TableCellType.CONTENT;
};
}
private Region toRegion(BoundingRegion boundingRegion, PageBatch batch) {
int pageNumber = batch.getPageNumber(boundingRegion.getPageNumber());
QuadPoint qp = QuadPoint.fromPolygons(boundingRegion.getPolygon()).getTransformed(getResultToPageTransform(pageNumber));
return new Region(pageNumber, qp.toData());
}
private Region toRegionFromRegions(int pageNumber, List<BoundingRegion> regions) {
if (regions.size() == 1) {
return new Region(pageNumber, QuadPoint.fromPolygons(regions.get(0).getPolygon()).getTransformed(getResultToPageTransform(pageNumber)).toData());
}
QuadPoint bbox = QuadPoint.fromRectangle2D(regions.stream()
.map(BoundingRegion::getPolygon)
.map(QuadPoint::fromPolygons)
.map(qp -> qp.getTransformed(getResultToPageTransform(pageNumber)).getBounds2D())
.collect(new Rectangle2DBBoxCollector()));
return new Region(pageNumber, bbox.toData());
}
private Region toRegionFromRegions(PageBatch batch, List<BoundingRegion> regions) {
assert !regions.isEmpty();
int batchPageNumber = regions.get(0).getPageNumber();
if (!regions.stream()
.map(BoundingRegion::getPageNumber)
.allMatch(number -> number == batchPageNumber)) {
throw new AssertionError();
}
int pageNumber = batch.getPageNumber(batchPageNumber);
return toRegionFromRegions(pageNumber, regions);
}
}

14
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/layers/LayerFactory.java
View File

@ -8,22 +8,19 @@ import java.util.Map;
import java.util.Set;
import com.azure.ai.documentintelligence.models.AnalyzeResult;
import com.knecon.fforesight.service.ocr.processor.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.model.PageBatch;
import com.knecon.fforesight.service.ocr.processor.model.PageInformation;
import com.knecon.fforesight.service.ocr.processor.service.OcrExecutionSupervisor;
import com.knecon.fforesight.service.ocr.processor.service.OcrResultPostProcessingPipeline;
import com.knecon.fforesight.service.ocr.processor.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.service.imageprocessing.ImageProcessingPipeline;
import com.knecon.fforesight.service.ocr.processor.visualizations.WritableOcrResult;
import com.knecon.fforesight.service.ocr.v1.api.model.IdpResult;
import com.knecon.fforesight.service.ocr.processor.service.OcrResultPostProcessingPipeline;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
import com.knecon.fforesight.service.viewerdoc.layers.LayerGroup;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class LayerFactory {
@ -32,7 +29,6 @@ public class LayerFactory {
IdpLayerFactory idpLayerFactory;
OcrDebugLayerFactory ocrDebugLayerFactory;
OcrTextLayerFactory ocrTextLayerFactory;
IdpResultFactory idpResultFactory;
OcrServiceSettings settings;
Set<AzureOcrFeature> features;
Map<Integer, Double> angles;
@ -52,13 +48,13 @@ public class LayerFactory {
this.features = features;
this.supervisor = supervisor;
this.angles = Collections.synchronizedMap(new HashMap<>());
this.idpResultFactory = new IdpResultFactory(ocrResultPostProcessingPipeline.getResultToPageTransforms(), pageInformation, angles, features);
}
public void processAnalyzeResult(PageBatch batch, AnalyzeResult analyzeResult) throws InterruptedException {
List<WritableOcrResult> results = ocrResultPostProcessingPipeline.processAnalyzeResult(analyzeResult, batch);
results.forEach(result -> angles.put(result.getPageNumber(), result.getAngle()));
ocrTextLayerFactory.addWritableOcrResult(results);
@ -68,7 +64,6 @@ public class LayerFactory {
}
if (features.contains(AzureOcrFeature.IDP)) {
idpLayerFactory.addAnalyzeResult(analyzeResult, batch);
idpResultFactory.addAnalyzeResult(analyzeResult, batch);
}
this.supervisor.finishMappingResult(batch);
@ -87,8 +82,7 @@ public class LayerFactory {
if (features.contains(AzureOcrFeature.IDP)) {
debugLayers.add(idpLayerFactory.getIdpLayer());
}
IdpResult idpResult = features.contains(AzureOcrFeature.IDP) ? idpResultFactory.getIdpResult() : null;
return new OcrResult(List.of(ocrTextLayer), debugLayers, angles, idpResult);
return new OcrResult(List.of(ocrTextLayer), debugLayers, angles);
}
}

3
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/layers/OcrResult.java
View File

@ -3,9 +3,8 @@ package com.knecon.fforesight.service.ocr.processor.visualizations.layers;
import java.util.List;
import java.util.Map;
import com.knecon.fforesight.service.ocr.v1.api.model.IdpResult;
import com.knecon.fforesight.service.viewerdoc.layers.LayerGroup;
public record OcrResult(List<LayerGroup> regularLayers, List<LayerGroup> debugLayers, Map<Integer, Double> anglesPerPage, IdpResult idpResult) {
public record OcrResult(List<LayerGroup> regularLayers, List<LayerGroup> debugLayers, Map<Integer, Double> anglesPerPage) {
}

40
azure-ocr-service/azure-ocr-service-processor/src/main/java/com/knecon/fforesight/service/ocr/processor/visualizations/utils/RotationCorrectionUtility.java
View File

@ -85,14 +85,12 @@ public class RotationCorrectionUtility {
List<String> commands = new LinkedList<>();
double scale = getScalingFactor(angle, page);
double x = page.getCropBox().getWidth() / 2;
double y = page.getCropBox().getHeight() / 2;
commands.add("q");
commands.add("/%s <<>> BDC".formatted(KNECON_ROTATION_CORRECTION.markedContentName()));
commands.add(buildMatrixCommands(AffineTransform.getTranslateInstance(x, y)));
commands.add(buildMatrixCommands(AffineTransform.getTranslateInstance(page.getPageWidth() / 2, page.getPageHeight() / 2)));
commands.add(buildMatrixCommands(AffineTransform.getRotateInstance(Math.toRadians(angle))));
commands.add(buildMatrixCommands(AffineTransform.getScaleInstance(scale, scale)));
commands.add(buildMatrixCommands(AffineTransform.getTranslateInstance(-x, -y)));
commands.add(buildMatrixCommands(AffineTransform.getTranslateInstance(-page.getPageWidth() / 2, -page.getPageHeight() / 2)));
commands.add("EMC");
return String.join("\n", commands);
}
@ -135,33 +133,23 @@ public class RotationCorrectionUtility {
public static AffineTransform buildTransform(double angle, double originalWidth, double originalHeight) {
return buildTransform(angle, originalWidth, originalHeight, true);
}
public static AffineTransform buildTransform(double angle, double originalWidth, double originalHeight, boolean quadrantRotation) {
int quadrants = getQuadrantRotation(angle);
double h = originalHeight;
double w = originalWidth;
AffineTransform quadrantRotationTransform = new AffineTransform();
if (quadrantRotation) {
if (quadrants == 1 || quadrants == 3) {
w = originalHeight;
h = originalWidth;
}
quadrantRotationTransform = switch (quadrants) {
case 1 -> new AffineTransform(0, 1, -1, 0, h, 0);
case 2 -> new AffineTransform(-1, 0, 0, -1, w, h);
case 3 -> new AffineTransform(0, -1, 1, 0, w - h, h);
default -> new AffineTransform();
};
if (quadrants == 1 || quadrants == 3) {
w = originalHeight;
h = originalWidth;
}
AffineTransform quadrantRotation = switch (quadrants) {
case 1 -> new AffineTransform(0, 1, -1, 0, h, 0);
case 2 -> new AffineTransform(-1, 0, 0, -1, w, h);
case 3 -> new AffineTransform(0, -1, 1, 0, w - h, h);
default -> new AffineTransform();
};
double remainder = getRemainingAngle(angle, quadrants);
double scale = getScalingFactor(remainder, w, h);
@ -170,7 +158,7 @@ public class RotationCorrectionUtility {
transform.rotate(Math.toRadians(remainder));
transform.scale(scale, scale);
transform.translate(-w / 2, -h / 2);
transform.concatenate(quadrantRotationTransform);
transform.concatenate(quadrantRotation);
return transform;
}
@ -199,7 +187,7 @@ public class RotationCorrectionUtility {
public static double getRemainingAngle(double angle, int quadrants) {
double referenceAngle = 90 * quadrants;
return (angle - referenceAngle) % 360;
return angle - referenceAngle;
}

2
azure-ocr-service/azure-ocr-service-processor/src/test/java/com/knecon/fforesight/service/ocr/processor/service/ImageProcessingPipelineTest.java
View File

@ -36,7 +36,7 @@ class ImageProcessingPipelineTest {
@BeforeEach
public void setup() {
new NativeLibrariesInitializer("demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a", "/home/kschuettler/software/leptonica/vcpkg/installed/x64-linux-dynamic/lib/").init();
new NativeLibrariesInitializer("demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a").init();
OcrServiceSettings settings = new OcrServiceSettings();
ImageProcessingService imageProcessingService = new ImageProcessingService(settings);

2
azure-ocr-service/azure-ocr-service-processor/src/test/java/com/knecon/fforesight/service/ocr/processor/service/SnugBoxesTest.java
View File

@ -56,7 +56,7 @@ public class SnugBoxesTest {
@BeforeAll
public static void setUp() {
new NativeLibrariesInitializer("demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a", "/home/kschuettler/software/leptonica/vcpkg/installed/x64-linux-dynamic/lib/").init();
new NativeLibrariesInitializer("demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a").init();
}

12
azure-ocr-service/azure-ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/FileStorageService.java
View File

@ -33,23 +33,20 @@ public class FileStorageService {
public void storeFiles(DocumentRequest request, File documentFile, File viewerDocumentFile, File analyzeResultFile) {
try (var in = new FileInputStream(viewerDocumentFile)) {
if (request.optionalViewerDocumentId()
.isPresent()) {
if (request.optionalViewerDocumentId().isPresent()) {
storageService.storeObject(TenantContext.getTenantId(), request.getViewerDocId(), in);
} else {
storageService.storeObject(TenantContext.getTenantId(), getStorageId(request.getDossierId(), request.getFileId(), FileType.VIEWER_DOCUMENT), in);
}
}
try (var in = new FileInputStream(documentFile)) {
if (request.optionalOriginDocumentId()
.isPresent()) {
if (request.optionalOriginDocumentId().isPresent()) {
storageService.storeObject(TenantContext.getTenantId(), request.getOriginDocumentId(), in);
} else {
storageService.storeObject(TenantContext.getTenantId(), getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN), in);
}
}
if (request.optionalIdpResultId()
.isPresent() && analyzeResultFile.exists()) {
if (request.optionalIdpResultId().isPresent()) {
try (var in = new FileInputStream(analyzeResultFile)) {
storageService.storeObject(TenantContext.getTenantId(), request.getIdpResultId(), in);
}
@ -62,8 +59,7 @@ public class FileStorageService {
Files.createDirectories(documentFile.getParentFile().toPath());
String originDocumentId = request.optionalOriginDocumentId()
.orElse(getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN));
String originDocumentId = request.optionalOriginDocumentId().orElse(getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN));
storageService.downloadTo(TenantContext.getTenantId(), originDocumentId, documentFile);

13
azure-ocr-service/azure-ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/NoStatusUpdateOcrMessageSender.java
View File

@ -1,13 +1,10 @@
package com.knecon.fforesight.service.ocr.v1.server.queue;
import java.util.Set;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
import com.knecon.fforesight.service.ocr.v1.server.configuration.MessagingConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -25,24 +22,24 @@ public class NoStatusUpdateOcrMessageSender implements IOcrMessageSender {
RabbitTemplate rabbitTemplate;
public void sendOcrFinished(String fileId, int totalImages, Set<AzureOcrFeature> features) {
public void sendOcrFinished(String fileId, int totalImages) {
}
public void sendOCRStarted(String fileId, Set<AzureOcrFeature> features) {
public void sendOCRStarted(String fileId) {
}
public void sendUpdate(String fileId, int finishedImages, int totalImages, Set<AzureOcrFeature> features) {
public void sendUpdate(String fileId, int finishedImages, int totalImages) {
}
public void sendOcrResponse(DocumentRequest request) {
public void sendOcrResponse(String dossierId, String fileId) {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_EXCHANGE, TenantContext.getTenantId(), request);
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_EXCHANGE, TenantContext.getTenantId(), new DocumentRequest(dossierId, fileId));
}
}

24
azure-ocr-service/azure-ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/OcrMessageReceiver.java
View File

@ -5,8 +5,6 @@ import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Objects;
import java.util.stream.Collectors;
import org.slf4j.MDC;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
@ -36,9 +34,6 @@ import lombok.extern.slf4j.Slf4j;
public class OcrMessageReceiver {
public static final String OCR_REQUEST_LISTENER_ID = "ocr-request-listener";
public static final String IDP_RESULT_FILE_NAME = "idpResult.json";
public static final String VIEWER_DOCUMENT_FILE_NAME = "viewerDocument.pdf";
public static final String DOCUMENT_FILE_NAME = "document.pdf";
FileStorageService fileStorageService;
ObjectMapper objectMapper;
@ -61,21 +56,21 @@ public class OcrMessageReceiver {
try {
MDC.put("fileId", fileId);
log.info("--------------------------------- Starting OCR ---------------------------------");
log.info("Features: {}", request.getFeatures().stream().map(Objects::toString).collect(Collectors.joining(", ")));
ocrMessageSender.sendOCRStarted(fileId, request.getFeatures());
log.info("--------------------------------------------------------------------------");
File documentFile = runDir.resolve(DOCUMENT_FILE_NAME).toFile();
File viewerDocumentFile = runDir.resolve(VIEWER_DOCUMENT_FILE_NAME).toFile();
File idpResultFile = runDir.resolve(IDP_RESULT_FILE_NAME).toFile();
ocrMessageSender.sendOCRStarted(fileId);
File documentFile = runDir.resolve("document.pdf").toFile();
File viewerDocumentFile = runDir.resolve("viewerDocument.pdf").toFile();
File analyzeResultFile = runDir.resolve("azureAnalysisResult.json").toFile();
fileStorageService.downloadFiles(request, documentFile);
ocrService.runOcrOnDocument(dossierId, fileId, request.getFeatures(), runDir, documentFile, viewerDocumentFile, idpResultFile);
ocrService.runOcrOnDocument(dossierId, fileId, request.getFeatures(), runDir, documentFile, viewerDocumentFile, analyzeResultFile);
fileStorageService.storeFiles(request, documentFile, viewerDocumentFile, idpResultFile);
fileStorageService.storeFiles(request, documentFile, viewerDocumentFile, analyzeResultFile);
ocrMessageSender.sendOcrResponse(request);
ocrMessageSender.sendOcrResponse(dossierId, fileId);
} catch (Exception e) {
log.warn("An exception occurred in ocr file stage: {}", e.getMessage());
in.getMessageProperties().getHeaders().put(MessagingConfiguration.X_ERROR_INFO_HEADER, e.getMessage());
@ -83,7 +78,6 @@ public class OcrMessageReceiver {
throw new RuntimeException(e);
} finally {
log.info("Done");
log.info("--------------------------------- Done ---------------------------------");
MDC.remove("fileId");
FileSystemUtils.deleteRecursively(runDir);
}

30
azure-ocr-service/azure-ocr-service-server/src/main/java/com/knecon/fforesight/service/ocr/v1/server/queue/OcrMessageSender.java
View File

@ -1,13 +1,10 @@
package com.knecon.fforesight.service.ocr.v1.server.queue;
import java.util.Set;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
import com.knecon.fforesight.service.ocr.v1.api.model.AzureOcrFeature;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
import com.knecon.fforesight.service.ocr.v1.api.model.OCRStatusUpdateResponse;
import com.knecon.fforesight.service.ocr.v1.server.configuration.MessagingConfiguration;
@ -28,46 +25,35 @@ public class OcrMessageSender implements IOcrMessageSender {
RabbitTemplate rabbitTemplate;
public void sendOcrFinished(String fileId, int totalImages, Set<AzureOcrFeature> features) {
public void sendOcrFinished(String fileId, int totalImages) {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_EXCHANGE,
TenantContext.getTenantId(),
OCRStatusUpdateResponse.builder()
.fileId(fileId)
.numberOfPagesToOCR(totalImages)
.numberOfOCRedPages(totalImages)
.ocrFinished(true)
.features(features)
.build());
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(totalImages).numberOfOCRedPages(totalImages).ocrFinished(true).build());
}
public void sendOCRStarted(String fileId, Set<AzureOcrFeature> features) {
public void sendOCRStarted(String fileId) {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_EXCHANGE,
TenantContext.getTenantId(),
OCRStatusUpdateResponse.builder().fileId(fileId).features(features).ocrStarted(true).build());
OCRStatusUpdateResponse.builder().fileId(fileId).ocrStarted(true).build());
}
public void sendUpdate(String fileId, int finishedImages, int totalImages, Set<AzureOcrFeature> features) {
public void sendUpdate(String fileId, int finishedImages, int totalImages) {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_EXCHANGE,
TenantContext.getTenantId(),
OCRStatusUpdateResponse.builder()
.fileId(fileId)
.features(features)
.numberOfPagesToOCR(totalImages)
.numberOfOCRedPages(finishedImages)
.build());
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(totalImages).numberOfOCRedPages(finishedImages).build());
}
public void sendOcrResponse(DocumentRequest request) {
public void sendOcrResponse(String dossierId, String fileId) {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_EXCHANGE, TenantContext.getTenantId(), request);
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_RESPONSE_EXCHANGE, TenantContext.getTenantId(), new DocumentRequest(dossierId, fileId));
}
}

2
azure-ocr-service/azure-ocr-service-server/src/main/resources/Aptfile
View File

@ -1,5 +1,5 @@
# you can list packages
ghostscript=9.55.0~dfsg1-0ubuntu5.10
ghostscript=9.55.0~dfsg1-0ubuntu5.9
pkg-config
zip
unzip

2
azure-ocr-service/azure-ocr-service-server/src/main/resources/application.yml
View File

@ -63,5 +63,3 @@ azure:
ocrService:
sendStatusUpdates: true
native-libs.path: ${VCPKG_DYNAMIC_LIB}

50
azure-ocr-service/azure-ocr-service-server/src/test/java/com/knecon/fforesight/service/ocr/v1/api/model/QuadPointTest.java
View File

@ -1,50 +0,0 @@
package com.knecon.fforesight.service.ocr.v1.api.model;
import static org.junit.jupiter.api.Assertions.*;
import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D;
import org.junit.jupiter.api.Test;
class QuadPointTest {
@Test
public void testContains() {
var a = new Point2D.Double(0, 0);
var b = new Point2D.Double(0, 1);
var c = new Point2D.Double(1, 1);
var d = new Point2D.Double(1, 0);
var q = new QuadPoint(a, b, c, d);
assertTrue(q.isHorizontal());
assertFalse(q.isVertical());
assertTrue(q.contains(a));
assertTrue(q.contains(b));
assertTrue(q.contains(c));
assertTrue(q.contains(d));
var p = new Point2D.Double(0.5, 0.5);
assertTrue(q.contains(p));
var r = new Rectangle2D.Double(0.5, 0.5, 0.1, 0.1);
assertTrue(q.contains(r));
}
@Test
public void testCenter() {
var a = new Point2D.Double(0, 0);
var b = new Point2D.Double(1, 1);
var c = new Point2D.Double(2, 1);
var d = new Point2D.Double(1, 0);
var q = new QuadPoint(a, b, c, d);
assertTrue(q.isHorizontal());
assertFalse(q.isVertical());
assertEquals(QuadPoint.Direction.RIGHT, q.getDirection());
assertEquals(new Point2D.Double(1, 0.5), q.getCenter());
}
}

13
azure-ocr-service/azure-ocr-service-server/src/test/java/com/knecon/fforesight/service/ocr/v1/server/OcrServiceIntegrationTest.java
View File

@ -1,9 +1,6 @@
package com.knecon.fforesight.service.ocr.v1.server;
import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
import static com.knecon.fforesight.service.ocr.v1.server.queue.OcrMessageReceiver.DOCUMENT_FILE_NAME;
import static com.knecon.fforesight.service.ocr.v1.server.queue.OcrMessageReceiver.IDP_RESULT_FILE_NAME;
import static com.knecon.fforesight.service.ocr.v1.server.queue.OcrMessageReceiver.VIEWER_DOCUMENT_FILE_NAME;
import java.io.File;
import java.io.FileInputStream;
@ -33,7 +30,7 @@ import lombok.SneakyThrows;
@SpringBootTest()
public class OcrServiceIntegrationTest extends AbstractTest {
public static final Set<AzureOcrFeature> FEATURES = Set.of(AzureOcrFeature.ROTATION_CORRECTION, AzureOcrFeature.FONT_STYLE_DETECTION, AzureOcrFeature.IDP);
public static final Set<AzureOcrFeature> FEATURES = Set.of(AzureOcrFeature.ROTATION_CORRECTION, AzureOcrFeature.FONT_STYLE_DETECTION);
@Autowired
private OCRService ocrService;
@ -58,7 +55,7 @@ public class OcrServiceIntegrationTest extends AbstractTest {
@SneakyThrows
public void testOcrWithFile() {
testOCR("/home/kschuettler/Dokumente/TestFiles/OCR/TestSet/VV-331340-first100.pdf");
testOCR("/home/kschuettler/Dokumente/TestFiles/OCR/VV-331340/VV-331340.pdf");
}
@ -111,9 +108,9 @@ public class OcrServiceIntegrationTest extends AbstractTest {
assert tmpDir.toFile().exists() || tmpDir.toFile().mkdirs();
var documentFile = tmpDir.resolve(Path.of(DOCUMENT_FILE_NAME));
var viewerDocumentFile = tmpDir.resolve(Path.of(VIEWER_DOCUMENT_FILE_NAME));
var analyzeResultFile = tmpDir.resolve(Path.of(IDP_RESULT_FILE_NAME));
var documentFile = tmpDir.resolve(Path.of("document.pdf"));
var viewerDocumentFile = tmpDir.resolve(Path.of("viewerDocument.pdf"));
var analyzeResultFile = tmpDir.resolve(Path.of("azureAnalysisResult.json"));
Files.copy(file.toPath(), documentFile, StandardCopyOption.REPLACE_EXISTING);
Files.copy(file.toPath(), viewerDocumentFile, StandardCopyOption.REPLACE_EXISTING);

9
azure-ocr-service/azure-ocr-service-server/src/test/resources/application.yml
View File

@ -2,10 +2,8 @@ persistence-service.url: "http://persistence-service-v1:8080"
pdftron.license: demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a
azure:
endpoint: https://ff-ocr-dev.cognitiveservices.azure.com/
key: 444fe2f83e9c48da8e588c7bd5295309 # find key in Bitwarden under: Azure IDP Test Key
native-libs:
endpoint: https://ff-ocr-test.cognitiveservices.azure.com/
key: # find key in Bitwarden under: Azure IDP Test Key
logging.type: ${LOGGING_TYPE:CONSOLE}
@ -21,5 +19,4 @@ management:
endpoints.web.exposure.include: prometheus, health, metrics
metrics.export.prometheus.enabled: true
POD_NAME: azure-ocr-service
native-libs.path: /home/kschuettler/software/leptonica/vcpkg/installed/x64-linux-dynamic/lib/
POD_NAME: azure-ocr-service

1
config/pmd/pmd.xml
View File

@ -15,7 +15,6 @@
<exclude name="NullAssignment"/>
<exclude name="AssignmentInOperand"/>
<exclude name="BeanMembersShouldSerialize"/>
<exclude name="AvoidFieldNameMatchingMethodName"/>
</rule>
</ruleset>

1
config/pmd/test_pmd.xml
View File

@ -17,7 +17,6 @@
<exclude name="AssignmentInOperand"/>
<exclude name="TestClassWithoutTestCases"/>
<exclude name="BeanMembersShouldSerialize"/>
<exclude name="AvoidFieldNameMatchingMethodName"/>
</rule>
</ruleset>