Compare commits

..
68 Commits
Author SHA1 Message Date
Dominique Eifländer 8924e905ad hotfix: Extend Tesseract instead of Tesseract1 2024-01-15 16:13:43 +01:00
Kilian Schüttler fb1fe35bc1 Merge branch 'RED-8155' into 'master'
RED-8155: bold-detection in ocr-service

Closes RED-8155

See merge request redactmanager/ocr-service!33
2024-01-08 13:53:30 +01:00
Kilian Schuettler 912f00aa84 RED-8155: bold-detection in ocr-service
* fix application.yml
2024-01-08 13:49:58 +01:00
Kilian Schüttler bab16ad9b2 Merge branch 'RED-7669-fontstyle' into 'master'
RED-8155: integrate bold-detection into ocr-service

Closes RED-7669

See merge request redactmanager/ocr-service!31
2024-01-05 16:05:53 +01:00
Kilian Schüttler be4656189b RED-8155: integrate bold-detection into ocr-service 2024-01-05 16:05:53 +01:00
Dominique Eifländer 8944b57344 Merge branch 'RED-7669' into 'master'
RED-7669: optimize OCR-module performance

Closes RED-7669

See merge request redactmanager/ocr-service!30
2023-12-22 15:14:42 +01:00
Kilian Schuettler 67540950b8 RED-7669: optimize OCR-module performance
* fix thread handling for PDFs without any images
2023-12-22 15:11:29 +01:00
Kilian Schuettler 6f29270e66 RED-7669: optimize OCR-module performance
* fix thread handling for PDFs without any images
2023-12-22 15:04:52 +01:00
Dominique Eifländer b961c9e324 Merge branch 'RED-1137' into 'master'
RED-1137: Do not observe actuator endpoints

Closes RED-1137

See merge request redactmanager/ocr-service!29
2023-12-20 14:24:49 +01:00
Dominique Eifländer 4b6411161e RED-1137: Do not observe actuator endpoints 2023-12-20 14:17:09 +01:00
Dominique Eifländer 14982eae7c Merge branch 'RED-5223' into 'master'
RED-5223: Use tracing-commons from fforesight

Closes RED-5223

See merge request redactmanager/ocr-service!28
2023-12-13 16:16:29 +01:00
Dominique Eifländer 99fc16130b RED-5223: Use tracing-commons from fforesight 2023-12-13 16:10:10 +01:00
Dominique Eifländer 80d38fb785 Merge branch 'RED-7669' into 'master'
RED-7669: optimize OCR-module performance

Closes RED-7669

See merge request redactmanager/ocr-service!27
2023-12-12 15:27:00 +01:00
Kilian Schüttler c06974ce69 RED-7669: optimize OCR-module performance 2023-12-12 15:27:00 +01:00
Dominique Eifländer 591c7d7fab Merge branch 'RED-5223' into 'master'
RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs

Closes RED-5223

See merge request redactmanager/ocr-service!26
2023-12-12 12:36:01 +01:00
Dominique Eifländer 0300a087d4 RED-5223: Enabled tracing, upgrade spring, use logstash-logback-encoder for json logs 2023-12-12 11:55:01 +01:00
Andrei Isvoran 98752ff1d1 Merge branch 'RED-7714' into 'master'
RED-7715 - Add log4j config to enable switching between json/line logs

Closes RED-7714

See merge request redactmanager/ocr-service!25
2023-12-06 12:42:33 +01:00
Andrei Isvoran ae09a59a7c RED-7715 - Add log4j config to enable switching between json/line logs 2023-12-06 11:52:01 +02:00
Kilian Schüttler 65d818200f Merge branch 'RED-7669' into 'master'
RED-7669: optimize OCR-module performance

Closes RED-7669

See merge request redactmanager/ocr-service!24
2023-11-28 12:35:22 +01:00
Kilian Schuettler 6fe95c6940 RED-7669: optimize OCR-module performance
* dont interrupt threads, use boolean flag instead
2023-11-28 10:04:56 +01:00
Kilian Schüttler 202132e14c Merge branch 'RED-7668' into 'master'
RED-7669: optimize OCR-module performance

Closes RED-7668

See merge request redactmanager/ocr-service!23
2023-11-24 10:57:45 +01:00
Kilian Schuettler 0264e28cc2 RED-7669: optimize OCR-module performance
* enable caches
2023-11-24 10:21:55 +01:00
Dominique Eifländer a50f54676e Merge branch 'RED-7668' into 'master'
RED-7669: optimize OCR-module performance

Closes RED-7668

See merge request redactmanager/ocr-service!22
2023-11-23 16:04:43 +01:00
Kilian Schuettler 1926707ae1 RED-7669: optimize OCR-module performance
* move all critical stuff to its own singleton thread
* make gs process queue any image once the file has been written
2023-11-23 16:00:53 +01:00
Kilian Schuettler d3190844a3 RED-7669: optimize OCR-module performance
* move all critical stuff to its own singleton thread
* make gs process queue any image once the file has been written
2023-11-23 16:00:31 +01:00
Kilian Schuettler c7ccbae6ff RED-7669: optimize OCR-module performance
* move all critical stuff to its own singleton thread
* make gs process queue any image once the file has been written
2023-11-23 16:00:31 +01:00
Kilian Schuettler 880bebcafc RED-7669: optimize OCR-module performance
* move all critical stuff to its own singleton thread
* make gs process queue any image once the file has been written
2023-11-23 16:00:31 +01:00
Kilian Schuettler 955ff6281d RED-7669: optimize OCR-module performance
* move all critical stuff to its own singleton thread
* make gs process queue any image once the file has been written
2023-11-23 16:00:31 +01:00
Kilian Schuettler efd3a1d952 RED-7669: optimize OCR-module performance
* move all non thread safe stuff to separate thread in the middle
2023-11-23 16:00:29 +01:00
Kilian Schuettler bb5b4a2fd8 RED-7669: optimize OCR-module performance
* binarize images after reading
2023-11-23 16:00:22 +01:00
Kilian Schuettler 6f99664906 RED-7669: optimize OCR-module performance
* try and synchronize all malloc calls
2023-11-23 16:00:19 +01:00
Kilian Schuettler 574f7ac25e RED-7669: optimize OCR-module performance
* moar sigsegv
2023-11-23 16:00:01 +01:00
Kilian Schuettler 12217f2459 RED-7669: optimize OCR-module performance
* moar sigsegv
2023-11-23 16:00:01 +01:00
Kilian Schuettler 19747cbca5 RED-7669: optimize OCR-module performance
* moar sigsegv
2023-11-23 15:59:59 +01:00
Kilian Schuettler 2632d2023d RED-7669: optimize OCR-module performance
* reset test and settings
2023-11-23 15:59:16 +01:00
Kilian Schuettler 4c225c2219 RED-7669: optimize OCR-module performance
* cleanup Code
2023-11-23 15:59:16 +01:00
Kilian Schuettler 3d09f46844 RED-7669: optimize OCR-module performance
* don't despeckle small images
2023-11-23 15:59:16 +01:00
Kilian Schuettler 77355b5367 RED-7669: optimize OCR-module performance
* second attempt at thread safety
2023-11-23 15:59:16 +01:00
Kilian Schuettler 57e194fcd0 RED-7669: optimize OCR-module performance
* attempt at thread safety
2023-11-23 15:59:14 +01:00
Kilian Schüttler c556687499 Merge branch 'RED-7669' into 'master'
RED-7669: optimize OCR-module performance

Closes RED-7669

See merge request redactmanager/ocr-service!21
2023-11-20 09:55:48 +01:00
Kilian Schüttler 759bae6499 RED-7669: optimize OCR-module performance 2023-11-20 09:55:48 +01:00
Kilian Schüttler aa45fa84bb Merge branch 'CYB-001' into 'master'
CYB-001: Improve OCR-Module performance

Closes CYB-001

See merge request redactmanager/ocr-service!20
2023-11-14 09:17:46 +01:00
Kilian Schüttler a82676c36b CYB-001: Improve OCR-Module performance 2023-11-14 09:17:46 +01:00
Kilian Schüttler 948c4bed79 Merge branch 'RED-7686' into 'master'
RED-7686 - Specific hidden text in specific file is not removed

Closes RED-7686

See merge request redactmanager/ocr-service!19
2023-10-16 09:27:34 +02:00
Corina Olariu 6d3ec8a9db RED-7686 - Specific hidden text in specific file is not removed
- upgraded storage-commons, tenant-commons to the newest windows compatible versions
2023-10-13 14:50:31 +03:00
Corina Olariu 6533501ffc RED-7686 - Specific hidden text in specific file is not removed
- update pdftron-logic-commons dependency
2023-10-13 13:45:44 +03:00
Raphael Arnold 607b9be6f5 Merge branch 'RED-7075' into 'master'
RED-7075: Watermark recognition improval

Closes RED-7075

See merge request redactmanager/ocr-service!18
2023-09-01 12:22:46 +02:00
RaphaelArnold acba4cb103 RED-7075: Watermark recognition improval 2023-09-01 12:13:22 +02:00
Andrei Isvoran bf3fe7f13d Merge branch 'RED-6864' into 'master'
RED-6864 - Fix storage update

Closes RED-6864

See merge request redactmanager/ocr-service!17
2023-08-18 09:22:27 +02:00
Andrei Isvoran ede443a47a RED-6864 - Fix storage update 2023-08-18 10:19:33 +03:00
Andrei Isvoran f5f1f70ffd Merge branch 'RED-6864' into 'master'
RED-6864 - Update ocr-service to new storage

Closes RED-6864

See merge request redactmanager/ocr-service!16
2023-08-16 11:43:53 +02:00
Andrei Isvoran 1c62c5ddf4 RED-6864 - Update ocr-service to new storage 2023-08-16 11:32:49 +03:00
deiflaender 506b888424 RED-7080: Fixed NPE in OCGWatermark removal 2023-08-14 16:35:31 +02:00
deiflaender 37ff2b982a RED-7080: Remove all watermarks that are named as watermarks in OCG 2023-08-14 16:11:48 +02:00
deiflaender 06c49cc412 RED-7080: Remove watermarks that are named as watermarks in OCG 2023-08-14 13:26:03 +02:00
deiflaender 262204bcca hotfix: Fixed npe for inline image where getXObject() returns null 2023-08-11 15:16:28 +02:00
Raphael Arnold 04a0925a6c Merge branch 'RED-7075' into 'master'
RED-7075: WIP

Closes RED-7075

See merge request redactmanager/ocr-service!15
2023-08-07 13:33:10 +02:00
RaphaelArnold 57ef7da5b3 RED-7075: WIP 2023-08-07 13:29:17 +02:00
Kilian Schuettler 7e20541d73 hotfix: fix OCRServiceIntegrationTest 2023-08-07 12:39:49 +02:00
Andrei Isvoran 33412589c0 RED-7290 - Update platform-common-dependency version 2023-08-03 18:14:46 +03:00
Andrei Isvoran 0ff07979ee Merge branch 'RED-7080' into 'master'
RED-7080 - Add removeWatermark flag for dossier template

Closes RED-7080

See merge request redactmanager/ocr-service!14
2023-08-01 08:27:00 +02:00
Andrei Isvoran 0ad4682571 RED-7080 - Add removeWatermark flag for dossier template 2023-07-31 15:51:22 +03:00
Timo Bejan 74f9f123f4 Update pom.xml 2023-07-27 08:49:44 +02:00
Ali Oezyetimoglu 7525a54341 Merge branch 'RED-7012' into 'master'
RED-7012: upgraded pdftron-logic-commons version

Closes RED-7012

See merge request redactmanager/ocr-service!13
2023-07-19 15:11:34 +02:00
Ali Oezyetimoglu 7209d47862 RED-7012: upgraded pdftron-logic-commons version 2023-07-19 15:06:22 +02:00
Kilian Schüttler 82db83936d Merge branch 'DM-326' into 'master'
DM-326: extend removeInvisibleElements

Closes DM-326

See merge request redactmanager/ocr-service!10
2023-07-14 12:48:49 +02:00
Kilian Schuettler 856d52951c DM-326: extend removeInvisibleElements 2023-07-14 12:46:24 +02:00
Kilian Schuettler 95af0faecd added watermark removal test 2023-07-13 11:25:16 +02:00
130 changed files with 4444 additions and 1672 deletions
+47
View File
@@ -9,6 +9,49 @@
**/tmp/
**/.apt_generated/
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### maven build ###
*.class
/out/
/build/
/target/
**/out/
**/build/
**/target/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
.gradle
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/
.factorypath
.springBeans
@@ -26,3 +69,7 @@
**/.DS_Store
**/classpath-data.json
**/dependencies-and-licenses-overview.txt
gradle.properties
gradlew
gradlew.bat
gradle/
+18 -3
View File
@@ -1,6 +1,21 @@
variables:
SONAR_PROJECT_KEY: 'LM_license-service'
include:
- project: 'gitlab/gitlab'
ref: 'main'
file: 'ci-templates/maven_java.yml'
file: 'ci-templates/gradle_java.yml'
deploy:
stage: deploy
tags:
- dind
script:
- echo "Building with gradle version ${BUILDVERSION}"
- gradle -Pversion=${BUILDVERSION} publish
- gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
- echo "BUILDVERSION=$BUILDVERSION" >> version.env
artifacts:
reports:
dotenv: version.env
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH =~ /^release/
- if: $CI_COMMIT_TAG
+85
View File
@@ -0,0 +1,85 @@
# OCR Service
## Overview
The OCR service is a tool designed for extracting text content from PDF files. It utilizes Tesseract, Leptonica, PDFTron, PDFBox, and Ghostscript to perform various tasks, including removing invisible elements and watermarks, extracting images, stitching striped images, binarizing images, running OCR on the processed images, and writing the recognized text back to the original PDF. This service is particularly useful for obtaining machine-readable text from PDF documents.
## Dependencies
[Tesseract](https://github.com/tesseract-ocr/tesseract)
[Leptonica](http://leptonica.org/)
[PDFTron](https://apryse.com/)
[PDFBox](https://pdfbox.apache.org/)
[Ghostscript](https://www.ghostscript.com/)
## Functionality
1. Invisible Element and Watermark Removal
The service uses PDFTron to attempt the removal of invisible elements and watermarks from the PDF.
2. Image Extraction
Extracts all images from the PDF using PDFBox
3. Striped Image Detection and Stitching
Detects if images are striped and stitches them together using Ghostscript.
4. Image Processing
- Convert to grayscale
- Upscale to target DPI
- Filter using Gauss kernel
- Binarizes the resulting images using Leptonica and the Otsu thresholding algorithm.
- Despeckle using various morphological operations
5. OCR Processing
Runs Tesseract on the images to extract text.
6. Font style detection
Detection of bold text using stroke width estimation
7. Text Integration
Draws the resulting text onto the original PDF using PDFBox.
Steps 2.-5. happen in parallel and communicate via a blocking queue to limit RAM usage.
Therefore, choosing your thread counts carefully leads to most optimal performance.
For example with 18 available cores, I achieved the highest performance with 2 Image extraction threads, 2 ghostscript processes and 16 OCR threads.
Setting all threads to basically unlimited (1000+) leads to comparable performance without laborious thread tuning, but at the cost of (potentially a lot) more RAM.
## Installation
To run the OCR service, ensure that the following dependencies are installed:
1. Ghostscript: Install using apt.
```bash
sudo apt install ghostscript
```
2. Tesseract and Leptonica: Install using [vcpkg](https://github.com/microsoft/vcpkg) with the command and set the environment variable `VCPKG_DYNAMIC_LIB` to your vcpkg lib folder (e.g. ~/vcpkg/installed/x64-linux-dynamic/lib).
```bash
vcpkg install tesseract --triplet x64-linux-dynamic
```
```bash
vcpkg install leptonica --triplet x64-linux-dynamic
```
3. Other dependencies are handled by Gradle build
```bash
gradle build
```
## Configuration
Configuration settings are available in the OcrServiceSettings class.
These settings can be overridden using environment variables. e.g.
`OCR_SERVICE_OCR_THREAD_COUNT=16`
Possible configurations and their defaults include:
```java
int ocrThreadCount = 4; // Number of OCR threads
int imageExtractThreadCount = 4; // Number of image extraction threads
int gsProcessCount = 4; // Number of Ghostscript processes
int dpi = 300; // Target DPI for binarized images
int psmOverride = -1; // Overrides the page segmentation mode if > 0
int minImageHeight = 20; // Minimum height for images to be processed
int minImageWidth = 20; // Minimum width for images to be processed
boolean debug = false; // If true, overlays OCR images with a grid and draws word bounding boxes
boolean removeWatermark; // If false, watermarks will not be removed
String languages = "deu+eng"; // Defines languages loaded into Tesseract as 3-char codes, additional languages must also be installed in the docker environment
```
## Integration
The OCR-service communicates via RabbitMQ and uses the queues `ocrQueue`, `ocrDLQ`, and `ocr_status_update_response_queue`.
### ocrQueue
This queue is used to start the OCR process, a DocumentRequest must be passed as a message. The service will then download the PDF from the provided cloud storage.
### ocr_status_update_response_queue
This queue is used by the OCR service to give updates about the progress of the ongoing OCR on a image per image basis. The total amount may change, when less images are found than initially assumed.
This queue is also used to signal the end of processing.
### ocrDLQ
This queue is used to signal an error has occurred during processing.
+15
View File
@@ -0,0 +1,15 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
// Support convention plugins written in Kotlin. Convention plugins are build scripts in 'src/main' that automatically become available as plugins in the main build.
`kotlin-dsl`
}
repositories {
// Use the plugin portal to apply community plugins in convention plugins.
gradlePluginPortal()
}
@@ -0,0 +1,66 @@
plugins {
`java-library`
pmd
checkstyle
jacoco
}
group = "com.knecon.fforesight.service"
java.sourceCompatibility = JavaVersion.VERSION_17
java.targetCompatibility = JavaVersion.VERSION_17
tasks.pmdMain {
pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
}
tasks.pmdTest {
pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
}
tasks.named<Test>("test") {
useJUnitPlatform()
reports {
junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
}
minHeapSize = "512m"
maxHeapSize = "8192m"
}
tasks.test {
finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
}
tasks.jacocoTestReport {
dependsOn(tasks.test) // tests are required to run before generating the report
reports {
xml.required.set(true)
csv.required.set(false)
html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
}
}
java {
withJavadocJar()
}
repositories {
mavenLocal()
maven {
url = uri("https://nexus.knecon.com/repository/gindev/")
credentials {
username = providers.gradleProperty("mavenUser").getOrNull()
password = providers.gradleProperty("mavenPassword").getOrNull()
}
}
maven {
url = uri("https://nexus.knecon.com/repository/PDFTron/")
credentials {
username = providers.gradleProperty("mavenUser").getOrNull()
password = providers.gradleProperty("mavenPassword").getOrNull()
}
}
mavenCentral()
}
+39
View File
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<module name="Checker">
<property
name="severity"
value="error"/>
<module name="TreeWalker">
<module name="SuppressWarningsHolder"/>
<module name="MissingDeprecated"/>
<module name="MissingOverride"/>
<module name="AnnotationLocation"/>
<module name="JavadocStyle"/>
<module name="NonEmptyAtclauseDescription"/>
<module name="IllegalImport"/>
<module name="RedundantImport"/>
<module name="RedundantModifier"/>
<module name="EmptyBlock"/>
<module name="DefaultComesLast"/>
<module name="EmptyStatement"/>
<module name="EqualsHashCode"/>
<module name="ExplicitInitialization"/>
<module name="IllegalInstantiation"/>
<module name="ModifiedControlVariable"/>
<module name="MultipleVariableDeclarations"/>
<module name="PackageDeclaration"/>
<module name="ParameterAssignment"/>
<module name="SimplifyBooleanExpression"/>
<module name="SimplifyBooleanReturn"/>
<module name="StringLiteralEquality"/>
<module name="OneStatementPerLine"/>
<module name="FinalClass"/>
<module name="ArrayTypeStyle"/>
<module name="UpperEll"/>
<module name="OuterTypeFilename"/>
</module>
<module name="FileTabCharacter"/>
<module name="SuppressWarningsFilter"/>
</module>
+20
View File
@@ -0,0 +1,20 @@
<?xml version="1.0"?>
<ruleset name="Custom ruleset"
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
<description>
Knecon ruleset checks the code for bad stuff
</description>
<rule ref="category/java/errorprone.xml">
<exclude name="DataflowAnomalyAnalysis"/>
<exclude name="MissingSerialVersionUID"/>
<exclude name="NullAssignment"/>
<exclude name="AvoidLiteralsInIfCondition"/>
<exclude name="AvoidDuplicateLiterals"/>
<exclude name="AvoidFieldNameMatchingMethodName"/>
</rule>
</ruleset>
+23
View File
@@ -0,0 +1,23 @@
<?xml version="1.0"?>
<ruleset name="Custom ruleset"
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
<description>
Knecon test ruleset checks the code for bad stuff
</description>
<rule ref="category/java/errorprone.xml">
<exclude name="DataflowAnomalyAnalysis"/>
<exclude name="MissingSerialVersionUID"/>
<exclude name="NullAssignment"/>
<exclude name="AvoidLiteralsInIfCondition"/>
<exclude name="AvoidDuplicateLiterals"/>
<exclude name="AvoidFieldNameMatchingMethodName"/>
<exclude name="AvoidFieldNameMatchingTypeName"/>
<exclude name="TestClassWithoutTestCases"/>
</rule>
</ruleset>
+1
View File
@@ -0,0 +1 @@
version = 4.0-SNAPSHOT
-118
View File
@@ -1,118 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.knecon.fforesight</groupId>
<artifactId>platform-docker-dependency</artifactId>
<version>0.1.0</version>
<relativePath/>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>ocr-service-image-v1</artifactId>
<groupId>com.iqser.red.service</groupId>
<version>3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<service.server>ocr-service-server-v1</service.server>
<platform.jar>${service.server}.jar</platform.jar>
<docker.skip.push>false</docker.skip.push>
<docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>download-platform-jar</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<artifactItems>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>${service.server}</artifactId>
<version>${version}</version>
<type>jar</type>
<overWrite>true</overWrite>
<destFileName>${platform.jar}</destFileName>
</dependency>
</artifactItems>
<outputDirectory>${docker.build.directory}</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
<configuration>
<images>
<image>
<name>${docker.image.name}</name>
<build>
<dockerFileDir>${docker.build.directory}</dockerFileDir>
<args>
<PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
</args>
<tags>
<tag>${docker.image.version}</tag>
<tag>latest</tag>
</tags>
</build>
</image>
</images>
</configuration>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-resources</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${basedir}/target/build/libs/</outputDirectory>
<resources>
<resource>
<directory>libs</directory>
<filtering>false</filtering>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
@@ -1,18 +0,0 @@
FROM red/base-image:2.0.2
COPY "libs/pdftron/OCRModuleLinux.tar.gz" .
RUN tar xvzf OCRModuleLinux.tar.gz
RUN mkdir /OCRModule
RUN mv Lib/* /OCRModule/
RUN apt-get -y update
# Ghostscript somehow improves ocr quality using pdfton, do not remove!
RUN apt-get -y install ghostscript
ARG PLATFORM_JAR
ENV PLATFORM_JAR ${PLATFORM_JAR}
ENV USES_ELASTICSEARCH false
COPY ["${PLATFORM_JAR}", "/"]
-33
View File
@@ -1,33 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.iqser.red.service</groupId>
<artifactId>ocr-service-v1</artifactId>
<version>3.0-SNAPSHOT</version>
</parent>
<artifactId>ocr-service-api-v1</artifactId>
<dependencies>
<dependency>
<!-- This dependency contains annotations that are used in specifying REST endpoints. -->
<!-- It is optional since not all users of this API might use Feign. -->
<groupId>io.github.openfeign</groupId>
<artifactId>feign-core</artifactId>
<optional>true</optional>
</dependency>
<!-- spring -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
</dependencies>
</project>
@@ -1,17 +0,0 @@
package com.iqser.red.service.ocr.v1.api.model;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
public class IdentityTest {
@Test
public void mockTest() {
int i = 1;
assertThat(i).isEqualTo(1);
}
}
@@ -0,0 +1,22 @@
plugins {
`maven-publish`
id("com.iqser.red.service.java-conventions")
id("io.freefair.lombok") version "8.2.2"
}
publishing {
publications {
create<MavenPublication>(name) {
from(components["java"])
}
}
repositories {
maven {
url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
credentials {
username = providers.gradleProperty("mavenUser").getOrNull();
password = providers.gradleProperty("mavenPassword").getOrNull();
}
}
}
}
@@ -1,13 +1,14 @@
package com.iqser.red.service.ocr.v1.api.model;
package com.knecon.fforesight.service.ocr.v1.api.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
@Data
@NoArgsConstructor
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DocumentRequest {
protected String dossierId;
@@ -1,4 +1,4 @@
package com.iqser.red.service.ocr.v1.api.model;
package com.knecon.fforesight.service.ocr.v1.api.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
@@ -6,9 +6,9 @@ import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class OCRStatusUpdateResponse {
private String fileId;
@@ -0,0 +1,29 @@
plugins {
id("com.iqser.red.service.java-conventions")
id("io.freefair.lombok") version "8.2.2"
}
configurations {
all {
exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
}
}
dependencies {
api("com.iqser.red.service:persistence-service-internal-api-v1:2.224.0")
api("net.sourceforge.tess4j:tess4j:5.8.0")
api("com.iqser.red.commons:metric-commons:2.1.0")
api("com.iqser.red.commons:storage-commons:2.45.0")
api("com.knecon.fforesight:tenant-commons:0.19.0")
api("com.pdftron:PDFNet:10.5.0")
api("org.apache.pdfbox:pdfbox:3.0.0")
api("org.apache.pdfbox:jbig2-imageio:3.0.4")
api("com.github.jai-imageio:jai-imageio-core:1.4.0")
api("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0")
api("org.apache.commons:commons-math3:3.6.1")
api("io.github.karols:hocr4j:0.2.0")
api("com.amazonaws:aws-java-sdk-kms:1.12.440")
api("com.google.guava:guava:31.1-jre")
api("com.iqser.red.commons:pdftron-logic-commons:2.20.0")
testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
}
@@ -0,0 +1,14 @@
package com.knecon.fforesight.service.ocr.processor;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
@Configuration
@ComponentScan
@EnableConfigurationProperties(OcrServiceSettings.class)
public class OcrServiceProcessorConfiguration {
}
@@ -1,6 +1,7 @@
package com.iqser.red.service.ocr.v1.server.initializer;
package com.knecon.fforesight.service.ocr.processor.initializer;
import com.pdftron.pdf.PDFNet;
import com.sun.jna.NativeLibrary;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
@@ -15,9 +16,6 @@ public class PDFNetInitializer {
@Value("${pdftron.license:}")
private String pdftronLicense;
@Value("${pdftron.ocrmodule.path:/tmp}")
private String ocrModulePath;
@SneakyThrows
@PostConstruct
@@ -25,7 +23,7 @@ public class PDFNetInitializer {
public void init() {
PDFNet.setTempPath("/tmp/pdftron");
PDFNet.addResourceSearchPath(ocrModulePath);
PDFNet.initialize(pdftronLicense);
System.setProperty("jna.library.path", System.getenv("VCPKG_DYNAMIC_LIB"));
}
}
@@ -0,0 +1,36 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.awt.geom.Rectangle2D;
import java.awt.image.BufferedImage;
import org.apache.pdfbox.pdmodel.graphics.color.PDColorSpace;
import org.apache.pdfbox.util.Matrix;
import com.knecon.fforesight.service.ocr.processor.utils.ImageProcessingUtils;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.util.LeptUtils;
public record ExtractedImage(
int pageNumber, QuadPoint position, int height, int width, BufferedImage image, Matrix ctm, int numberOnPage, PDColorSpace colorSpace) implements UnprocessedImage {
@SneakyThrows
public Pix asPix() {
BufferedImage image = ImageProcessingUtils.convertToDeviceColorSpace(this);
ImageProcessingUtils.setAlphaChannelToWhite(image);
return LeptUtils.convertImageToPix(image);
}
public QuadPoint getImageCoordinatesInInitialUserSpace() {
return QuadPoint.fromRectangle2D(new Rectangle2D.Double(0, 0, 1, 1)).getTransformed(ctm.createAffineTransform());
}
}
@@ -0,0 +1,61 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.awt.Graphics;
import java.awt.geom.AffineTransform;
import java.awt.image.BufferedImage;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceGray;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.util.Matrix;
import com.knecon.fforesight.service.ocr.processor.service.threads.OCRThread;
import com.knecon.fforesight.service.ocr.processor.utils.ImageProcessingUtils;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.tess4j.ITessAPI;
@Slf4j
@Getter
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ExtractedOcrImage implements OcrImage {
int pageNumber;
int numberOnPage;
int originalHeight;
int originalWidth;
Matrix ctm;
Pix pix;
int height;
int width;
int rotationDegrees;
@Override
public AffineTransform getImageCTM() {
AffineTransform affineTransform = ctm.createAffineTransform();
affineTransform.scale((double) 1 / getWidth(), (double) 1 / getHeight());
AffineTransform deRotationMatrix = switch (360 - rotationDegrees) {
case 90 -> new AffineTransform(0, 1, -1, 0, getHeight(), 0);
case 180 -> new AffineTransform(-1, 0, 0, -1, getWidth(), getHeight());
case 270 -> new AffineTransform(0, -1, 1, 0, getWidth() - getHeight(), getHeight()); // results from 90 + 180 rotations
default -> new AffineTransform();
};
affineTransform.concatenate(deRotationMatrix);
AffineTransform mirrorTransform = new AffineTransform(1, 0, 0, -1, 0, getHeight());
affineTransform.concatenate(mirrorTransform);
return affineTransform;
}
}
@@ -0,0 +1,17 @@
package com.knecon.fforesight.service.ocr.processor.model;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Getter
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public final class FontMetrics {
float descent; // descent is the part of the text which is below the baseline, e.g. the lower curve of a 'g'. https://en.wikipedia.org/wiki/Body_height_(typography)
float fontSize;
float heightScaling;
}
@@ -0,0 +1,5 @@
package com.knecon.fforesight.service.ocr.processor.model;
public record HeightAndDescent(float height, float descent) {
}
@@ -0,0 +1,127 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.awt.geom.AffineTransform;
import java.awt.geom.Point2D;
import java.awt.image.BufferedImage;
import com.knecon.fforesight.service.ocr.processor.service.threads.OCRThread;
import com.knecon.fforesight.service.ocr.processor.utils.PdfDpiCalculator;
import lombok.SneakyThrows;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.util.LeptUtils;
import net.sourceforge.tess4j.ITessAPI;
public interface OcrImage {
/**
* Retrieves the page number where the OCR image is located. It uses 1-based-index.
*
* @return The page number where the OCR image is located.
*/
int getPageNumber();
/**
* Retrieves the number of this image on the page. For full page images this always returns 0.
*
* @return The number of this image on the page.
*/
int getNumberOnPage();
/**
* Retrieves the height of the original image (not necessarily in pdf coordinates).
*
* @return the height of the image
*/
int getHeight();
/**
* Retrieves the width of the original image (not necessarily in pdf coordinates).
*
* @return the width of the image
*/
int getWidth();
/**
* Gets the outer boundary of the image in image coordinates. (0,0) is upper left corner. And height and width is the image size
*
* @return the QuadPoint representing the size of the image
*/
default QuadPoint getImageBounds() {
// cannot be solved with a nice rotation matrix. After rotating the text coordinates in the image will always start at (0,0) and will therefore always start at (0,0) in the PDF.
// So in order to mimic this behavior we need to start with (0,0) coordinates always.
if (getRotationDegrees() == 90 || getRotationDegrees() == 270) {
return new QuadPoint(new Point2D.Double(0, 0), new Point2D.Double(0, getWidth()), new Point2D.Double(getHeight(), getWidth()), new Point2D.Double(getHeight(), 0));
} else {
return new QuadPoint(new Point2D.Double(0, 0), new Point2D.Double(0, getHeight()), new Point2D.Double(getWidth(), getHeight()), new Point2D.Double(getWidth(), 0));
}
}
/**
* Retrieves the image coordinates in the PDF by transforming the image bounds using the current transformation matrix (CTM).
*
* @return The image coordinates as a QuadPoint object.
*/
default QuadPoint getImageCoordinatesInInitialUserSpace() {
return getImageBounds().getTransformed(getImageCTM());
}
/**
* Retrieves the rotation degree of the OCR image.
*
* @return The rotation degree of the OCR image.
*/
int getRotationDegrees();
/**
* Retrieves the optimal page segmentation mode for the OCR image.
*
* @return The optimal page segmentation mode.
*/
default int getOptimalPageSegmentationMode() {
if (getWidth() < 200 || getHeight() < 200) {
return ITessAPI.TessPageSegMode.PSM_SINGLE_BLOCK;
}
return ITessAPI.TessPageSegMode.PSM_AUTO;
} // TODO: evaluate if PSM can be dynamically chosen to increase performance
/**
* Retrieves the buffered image associated with the OCR image.
*
* @return The BufferedImage object representing the image.
*/
Pix getPix();
default int getDpi() {
return PdfDpiCalculator.calculateDpi(getImageBounds(), getImageCTM(), getWidth());
}
/**
* Retrieves the current transformation matrix (CTM). The CTM may be used to transform the image coordinates to Initial User Space coordinates.
*
* @return The AffineTransform representing the current transformation matrix.
*/
AffineTransform getImageCTM();
default void destroyPix() {
LeptUtils.disposePix(getPix());
}
}
@@ -0,0 +1,23 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.awt.geom.AffineTransform;
import java.util.List;
import com.knecon.fforesight.service.ocr.processor.service.HOcrPageParser;
import io.github.karols.hocr4j.Word;
public record OcrResult(OcrImage image, String tesseractOutputFilePath) {
public static OcrResult create(OcrImage image, String tesseractResult) {
return new OcrResult(image, tesseractResult);
}
public List<Word> getAllWords() {
return HOcrPageParser.extractHocrPage(tesseractOutputFilePath).getAllWords();
}
}
@@ -0,0 +1,35 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.knecon.fforesight.service.ocr.processor.model.scriptdetection.FontStyleDetectionModel;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontMetricsFactory;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontStyle;
public record OcrResultToWrite(List<TextPositionInImage> textPositionInImage, QuadPoint imageBoundingBox) {
public static OcrResultToWrite fromFontStyleDetectionModel(FontStyleDetectionModel fontStyleDetectionModel) {
return new OcrResultToWrite(fontStyleDetectionModel.getTextPositionInImages(), fontStyleDetectionModel.getImageBounds());
}
public static Map<Integer, List<OcrResultToWrite>> buildOcrResultsToWrite(List<OcrResult> ocrResults, FontMetricsFactory fontMetricsFactory) {
return ocrResults.stream()
.collect(Collectors.groupingBy(ocrResult -> ocrResult.image().getPageNumber()))
.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey,
entry -> entry.getValue()
.stream()
.map(ocrResult -> new OcrResultToWrite(ocrResult.getAllWords()
.stream()
.filter(word -> !word.isBlank())
.map(word -> new TextPositionInImage(word, ocrResult.image().getImageCTM(), fontMetricsFactory, FontStyle.REGULAR))
.toList(), ocrResult.image().getImageCoordinatesInInitialUserSpace()))
.toList()));
}
}
@@ -0,0 +1,12 @@
package com.knecon.fforesight.service.ocr.processor.model;
import org.apache.pdfbox.pdmodel.PDPage;
public record PageInformation(int height, int width, int number, int rotationDegrees) {
public static PageInformation fromPDPage(int pageNum, PDPage page) {
return new PageInformation((int) page.getMediaBox().getHeight(), (int) page.getMediaBox().getWidth(), pageNum, page.getRotation());
}
}
@@ -0,0 +1,106 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.awt.geom.AffineTransform;
import java.awt.geom.Line2D;
import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D;
import java.util.List;
import io.github.karols.hocr4j.Bounds;
public record QuadPoint(Point2D a, Point2D b, Point2D c, Point2D d) {
/*
B _____ C
| |
A|_____|D
*/
public static QuadPoint fromRectangle2D(Rectangle2D rectangle2D) {
return new QuadPoint(new Point2D.Double(rectangle2D.getX(), rectangle2D.getY()),
new Point2D.Double(rectangle2D.getX(), rectangle2D.getMaxY()),
new Point2D.Double(rectangle2D.getMaxX(), rectangle2D.getMaxY()),
new Point2D.Double(rectangle2D.getMaxX(), rectangle2D.getY()));
}
public static QuadPoint fromBounds(Bounds bounds) {
return new QuadPoint(new Point2D.Double(bounds.getLeft(), bounds.getBottom()),
new Point2D.Double(bounds.getLeft(), bounds.getTop()),
new Point2D.Double(bounds.getRight(), bounds.getTop()),
new Point2D.Double(bounds.getRight(), bounds.getBottom()));
}
public QuadPoint getTransformed(AffineTransform at) {
return new QuadPoint(at.transform(a, null), at.transform(b, null), at.transform(c, null), at.transform(d, null));
}
/**
* Determines if the given QuadPoint aligns with this QuadPoint within a given threshold.
* It does os by trying every possible combination of aligning sides. It starts with the most likely combination of ab and cd.
*
* @param other The QuadPoint to compare with.
* @param threshold The maximum distance allowed for alignment.
* @return True if the QuadPoints align within the threshold, false otherwise.
*/
public boolean aligns(QuadPoint other, double threshold) {
Line2D ab = new Line2D.Double(a, b);
Line2D bc = new Line2D.Double(b, c);
Line2D cd = new Line2D.Double(c, d);
Line2D da = new Line2D.Double(d, a);
Line2D ab2 = new Line2D.Double(other.a, other.b);
Line2D bc2 = new Line2D.Double(other.b, other.c);
Line2D cd2 = new Line2D.Double(other.c, other.d);
Line2D da2 = new Line2D.Double(other.d, other.a);
List<Line2D> lines = List.of(ab, cd, bc, da);
List<Line2D> lines2 = List.of(cd2, ab2, bc2, da2);
return lines.stream().anyMatch(line -> lines2.stream().anyMatch(line2 -> aligns(line, line2, threshold)));
}
private static boolean aligns(Line2D a, Line2D b, double threshold) {
return aligns(a.getP1(), a.getP2(), b.getP1(), b.getP2(), threshold);
}
private static boolean aligns(Point2D a, Point2D b, Point2D a2, Point2D b2, double threshold) {
if (a.distance(a2) < threshold && b.distance(b2) < threshold) {
return true;
}
return a.distance(b2) < threshold && b.distance(a2) < threshold;
}
@Override
public String toString() {
return String.format("A:(%.2f, %.2f) | B:(%.2f, %.2f) | C:(%.2f, %.2f) | D:(%.2f, %.2f)",
a().getX(),
a().getY(),
b().getX(),
b().getY(),
c().getX(),
c().getY(),
d().getX(),
d().getY());
}
public double size() {
return a().distance(b()) * a().distance(d());
}
}
@@ -0,0 +1,14 @@
package com.knecon.fforesight.service.ocr.processor.model;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
public record RenderedPageImageFile(int pageNumber, String absoluteFilePath) implements UnprocessedImage {
@Override
public Pix asPix() {
return Leptonica1.pixRead(absoluteFilePath);
}
}
@@ -0,0 +1,90 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.awt.geom.AffineTransform;
import java.awt.geom.Point2D;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.tess4j.ITessAPI;
@Getter
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class RenderedPageOcrImage implements OcrImage {
int height;
int width;
PageInformation pageInformation;
Pix pix;
int rotationDegrees;
@Override
public AffineTransform getImageCTM() {
double scalingFactor = calculateScalingFactor();
AffineTransform imageToCropBoxScaling = new AffineTransform(scalingFactor, 0, 0, scalingFactor, 0, 0);
AffineTransform mirrorMatrix = new AffineTransform(1, 0, 0, -1, 0, pageInformation.height());
AffineTransform rotationMatrix = switch (calculateTotalRotation()) {
case 90 -> new AffineTransform(0, 1, -1, 0, pageInformation.height(), 0);
case 180 -> new AffineTransform(-1, 0, 0, -1, pageInformation.width(), pageInformation.height());
case 270 -> new AffineTransform(0, -1, 1, 0, pageInformation.width() - pageInformation.height(), pageInformation.height()); // results from 90 + 180 rotations
default -> new AffineTransform();
};
// matrix multiplication is performed from right to left, so the order is reversed.
// scaling -> mirror -> rotation
AffineTransform resultMatrix = new AffineTransform();
resultMatrix.concatenate(rotationMatrix);
resultMatrix.concatenate(mirrorMatrix);
resultMatrix.concatenate(imageToCropBoxScaling);
return resultMatrix;
}
private int calculateTotalRotation() {
return (pageInformation.rotationDegrees() + (360 - rotationDegrees)) % 360;
}
@Override
public int getPageNumber() {
return pageInformation.number();
}
@Override
public int getNumberOnPage() {
return 0;
}
private double calculateScalingFactor() {
// PDFBox always returns page height and width based on rotation
double pageWidth;
if (pageInformation.rotationDegrees() == 90 || pageInformation.rotationDegrees() == 270) {
pageWidth = pageInformation.height();
} else {
pageWidth = pageInformation.width();
}
return pageWidth / width;
}
}
@@ -0,0 +1,135 @@
package com.knecon.fforesight.service.ocr.processor.model;
import java.awt.geom.AffineTransform;
import java.awt.geom.Point2D;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.util.Matrix;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontMetricsFactory;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontStyle;
import io.github.karols.hocr4j.Word;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.FieldDefaults;
@Getter
@FieldDefaults(level = AccessLevel.PRIVATE)
public class TextPositionInImage {
final QuadPoint position;
final String text;
final AffineTransform imageCTM;
@Setter
FontMetricsFactory fontMetricsFactory;
@Setter
FontStyle fontStyle;
public TextPositionInImage(Word word, AffineTransform imageCTM, FontMetricsFactory fontMetricsFactory, FontStyle fontStyle) {
this.position = QuadPoint.fromBounds(word.getBounds());
this.text = word.getText();
this.imageCTM = imageCTM;
this.fontMetricsFactory = fontMetricsFactory;
this.fontStyle = fontStyle;
}
public QuadPoint getTransformedTextBBox() {
return position.getTransformed(imageCTM);
}
public PDFont getFont() {
return fontMetricsFactory.getFont();
}
public Matrix getTextMatrix() {
FontMetrics metrics = fontMetricsFactory.calculateMetrics(text, getTransformedWidth(), getTransformedHeight());
// Matrix multiplication is from right to left:
// convert to image coords -> subtract descent -> scale height -> reverse imageCTM scaling -> translate to coordinates in image -> convert to pdf coords
// width must not be set, since it is scaled with the fontsize attribute
AffineTransform ctm = new AffineTransform();
ctm.concatenate(imageCTM);
ctm.translate(position.a().getX(), position.a().getY());
ctm.scale(getWidth() / getTransformedWidth(),
getHeight() / getTransformedHeight()); // scale with transformation coefficient, such that fontsize may be set with transformed width.
ctm.scale(1, metrics.getHeightScaling());
ctm.translate(0, metrics.getDescent());
ctm.concatenate(new AffineTransform(1, 0, 0, -1, 0, 0)); // start in image coordinates, with (0,0) being top left and negative height.
return new Matrix(ctm);
}
public double getFontSize() {
return fontMetricsFactory.calculateFontSize(text, getTransformedWidth());
}
public double getTransformedWidth() {
return transformedA().distance(transformedD());
}
public double getTransformedHeight() {
return transformedA().distance(transformedB());
}
public double getWidth() {
return position.a().distance(position.d());
}
public double getTextHeight() {
var metrics = fontMetricsFactory.calculateMetrics(text, getTransformedWidth(), getTransformedHeight());
return fontMetricsFactory.calculateFontSize(text, getTransformedWidth()) * metrics.getHeightScaling();
}
public double getHeight() {
return position.a().distance(position.b());
}
public Point2D transformedA() {
return imageCTM.transform(position.a(), null);
}
public Point2D transformedB() {
return imageCTM.transform(position.b(), null);
}
public Point2D transformedC() {
return imageCTM.transform(position.c(), null);
}
public Point2D transformedD() {
return imageCTM.transform(position.d(), null);
}
}
@@ -0,0 +1,9 @@
package com.knecon.fforesight.service.ocr.processor.model;
import net.sourceforge.lept4j.Pix;
public interface UnprocessedImage {
Pix asPix();
}
@@ -0,0 +1,58 @@
package com.knecon.fforesight.service.ocr.processor.model.scriptdetection;
import java.util.List;
import com.knecon.fforesight.service.ocr.processor.model.OcrImage;
import com.knecon.fforesight.service.ocr.processor.model.OcrResult;
import com.knecon.fforesight.service.ocr.processor.model.QuadPoint;
import com.knecon.fforesight.service.ocr.processor.model.TextPositionInImage;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontMetricsFactory;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.util.LeptUtils;
@Getter
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public final class FontStyleDetectionModel {
QuadPoint imageBounds;
Pix image;
List<TextPositionAndWordImage> textPositionsAndWordImages;
public static FontStyleDetectionModel fromOcrResult(OcrResult ocrResult, FontMetricsFactory fontMetricsFactory, OcrServiceSettings settings) {
var image = Leptonica1.pixRead(ocrResult.tesseractOutputFilePath() + ".tiff");
var wordPixes = ocrResult.getAllWords().stream().filter(word -> !word.isBlank()).map(word -> TextPositionAndWordImage.create(ocrResult.image().getImageCTM(), word, image, settings, fontMetricsFactory)).toList();
return new FontStyleDetectionModel(ocrResult.image().getImageCoordinatesInInitialUserSpace(), image, wordPixes);
}
public List<TextPositionInImage> getTextPositionInImages() {
return textPositionsAndWordImages.stream().map(TextPositionAndWordImage::getTextPositionInImage).toList();
}
public List<WordImage> getWordImages() {
return textPositionsAndWordImages.stream().map(TextPositionAndWordImage::getWordImage).toList();
}
public void dispose() {
LeptUtils.disposePix(image);
getWordImages().forEach(WordImage::dispose);
}
}
@@ -0,0 +1,52 @@
package com.knecon.fforesight.service.ocr.processor.model.scriptdetection;
import java.awt.geom.AffineTransform;
import java.util.Objects;
import org.apache.commons.math3.ml.clustering.Clusterable;
import com.knecon.fforesight.service.ocr.processor.model.OcrResult;
import com.knecon.fforesight.service.ocr.processor.model.TextPositionInImage;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontMetricsFactory;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontStyle;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import io.github.karols.hocr4j.Word;
import lombok.Getter;
import net.sourceforge.lept4j.Pix;
@Getter
public final class TextPositionAndWordImage implements Clusterable {
private final TextPositionInImage textPositionInImage;
private final WordImage wordImage;
public TextPositionAndWordImage(TextPositionInImage textPositionInImage, WordImage wordImage) {
this.textPositionInImage = textPositionInImage;
this.wordImage = wordImage;
}
public static TextPositionAndWordImage create(AffineTransform imageCTM, Word word, Pix image, OcrServiceSettings settings, FontMetricsFactory fontMetricsFactory) {
TextPositionInImage textPositionInImage = new TextPositionInImage(word, imageCTM, fontMetricsFactory, FontStyle.REGULAR);
WordImage wordImage = new WordImage(textPositionInImage.getTextHeight(), word, image, settings);
return new TextPositionAndWordImage(textPositionInImage, wordImage);
}
@Override
public double[] getPoint() {
return wordImage.getPoint();
}
public double getTextHeight() {
return wordImage.getTextHeight();
}
}
@@ -0,0 +1,71 @@
package com.knecon.fforesight.service.ocr.processor.model.scriptdetection;
import org.apache.commons.math3.ml.clustering.Clusterable;
import com.knecon.fforesight.service.ocr.processor.service.scriptdetection.StrokeWidthCalculator;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.utils.ImageProcessingUtils;
import io.github.karols.hocr4j.Word;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import net.sourceforge.lept4j.Box;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.util.LeptUtils;
@Getter
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class WordImage implements Clusterable {
Pix image;
String text;
double textHeight;
OcrServiceSettings settings;
public WordImage(double textHeight, Word word, Pix originalImage, OcrServiceSettings settings) {
Box box = new Box(word.getBounds().getLeft(), word.getBounds().getTop(), word.getBounds().getWidth(), word.getBounds().getHeight(), 1);
this.image = Leptonica1.pixClipRectangle(originalImage, box, null);
box.clear();
this.text = word.getText();
this.textHeight = textHeight;
this.settings = settings;
}
public boolean hasLargerStrokeWidth(double strokeWidth) {
int roundedStrokeWidth = (int) Math.round(strokeWidth);
double roundingError = (roundedStrokeWidth - strokeWidth) / strokeWidth;
// add 1 to open a bit bigger than the estimated regular stroke width
Pix openedPix = Leptonica1.pixOpenBrick(null, image, roundedStrokeWidth + 1, roundedStrokeWidth + 1);
double openedPixelDensity = ImageProcessingUtils.calculatePixelDensity(openedPix);
double pixelDensity = ImageProcessingUtils.calculatePixelDensity(image);
LeptUtils.disposePix(openedPix);
return (openedPixelDensity * (1 + roundingError)) / pixelDensity > (settings.getBoldThreshold());
}
@Override
public double[] getPoint() {
return new double[]{textHeight};
}
public void dispose() {
LeptUtils.disposePix(image);
}
}
@@ -1,12 +1,15 @@
package com.iqser.red.service.ocr.v1.server.service;
package com.knecon.fforesight.service.ocr.processor.service;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.ocr.v1.server.model.image.ImageServiceResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -32,14 +35,16 @@ public class FileStorageService {
@SneakyThrows
public byte[] getOriginalFile(String dossierId, String fileId) {
return IOUtils.toByteArray(storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream());
try (InputStream inputStream = getInputStream(getStorageId(dossierId, fileId, FileType.ORIGIN))) {
return IOUtils.toByteArray(inputStream);
}
}
@SneakyThrows
public InputStream getOriginalFileAsStream(String dossierId, String fileId) {
return storageService.getObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.ORIGIN)).getInputStream();
return getInputStream(getStorageId(dossierId, fileId, FileType.ORIGIN));
}
@@ -62,9 +67,11 @@ public class FileStorageService {
@SneakyThrows
public ImageServiceResponse getImageServiceResponse(String dossierId, String fileId) {
private InputStream getInputStream(String storageId) {
return storageService.readJSONObject(TenantContext.getTenantId(), getStorageId(dossierId, fileId, FileType.IMAGE_INFO), ImageServiceResponse.class);
File tempFile = File.createTempFile("temp", ".data");
storageService.downloadTo(TenantContext.getTenantId(), storageId, tempFile);
return Files.newInputStream(Paths.get(tempFile.getPath()), StandardOpenOption.DELETE_ON_CLOSE);
}
}
@@ -0,0 +1,169 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.stream.Collectors;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.ocr.processor.model.RenderedPageImageFile;
import com.knecon.fforesight.service.ocr.processor.model.UnprocessedImage;
import com.knecon.fforesight.service.ocr.processor.service.threads.BlockingQueueFiller;
import com.knecon.fforesight.service.ocr.processor.service.threads.GhostScriptOutputHandler;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.utils.ListSplittingUtils;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@SuppressWarnings("PMD") // can't figure out how to safely close the stdOut and stdError streams in line 142/144
public class GhostScriptService {
static String FORMAT = ".tiff";
static String DEVICE = "tiffgray";
OcrServiceSettings settings;
@SneakyThrows
public void renderPagesAsImagesBatchedAndAddToQueue(List<Integer> stitchedPageNumbers,
String documentAbsolutePath,
Path tmpImageDir,
PDDocument document,
BlockingQueue<UnprocessedImage> imageProcessingQueue,
Statistics stats) {
BlockingQueue<RenderedPageImageFile> imageFileCollectorQueue = new LinkedBlockingDeque<>();
BlockingQueueFiller asyncTransferThread = new BlockingQueueFiller(imageFileCollectorQueue, imageProcessingQueue);
asyncTransferThread.start();
int numOfProcesses = Math.min(settings.getGsProcessCount(), stitchedPageNumbers.size());
List<List<ProcessInfo>> processInfoBatches = buildSubListForEachProcess(stitchedPageNumbers,
numOfProcesses,
256 * numOfProcesses); // GS has a limit on how many pageIndices per call are possible, so we limit it to 256 pages per process
for (int batchIdx = 0; batchIdx < processInfoBatches.size(); batchIdx++) {
long timestamp = System.currentTimeMillis();
List<ProcessInfo> processInfos = processInfoBatches.get(batchIdx);
log.info("Batch {}: Running {} gs processes with ({}) pages each",
batchIdx,
processInfos.size(),
processInfos.stream().map(info -> info.stitchedPageNumbers().size()).map(String::valueOf).collect(Collectors.joining(", ")));
int finalBatchIdx = batchIdx;
List<Process> processes = processInfos.stream()
.parallel()
.map(info -> buildCmdArgs(info.processIdx(), finalBatchIdx, info.stitchedPageNumbers(), tmpImageDir, documentAbsolutePath))
.peek(s -> log.debug(String.join(" ", s.cmdArgs())))
.map(processInfo -> executeProcess(processInfo, imageFileCollectorQueue))
.toList();
List<Integer> processExitCodes = new LinkedList<>();
for (Process process : processes) {
processExitCodes.add(process.waitFor());
}
stats.increasePDF2ImgDuration(System.currentTimeMillis() - timestamp);
log.info("Batch {}: Ghostscript processes finished with exit codes " + processExitCodes, batchIdx);
}
asyncTransferThread.setAllImagesQueued(true);
}
private List<List<ProcessInfo>> buildSubListForEachProcess(List<Integer> stitchedPageNumbers, int processCount, int batchSize) {
// GhostScript command line can only handle so many page numbers at once, so we split it into batches
int batchCount = (int) Math.ceil((double) stitchedPageNumbers.size() / batchSize);
log.info("Splitting {} page renderings across {} process(es) in {} batch(es) with size {}", stitchedPageNumbers.size(), processCount, batchCount, batchSize);
List<List<ProcessInfo>> processInfoBatches = new ArrayList<>(batchCount);
List<List<List<Integer>>> batchedBalancedSublist = ListSplittingUtils.buildBatchedBalancedSublist(stitchedPageNumbers.stream().sorted().toList(), processCount, batchCount);
for (var batch : batchedBalancedSublist) {
List<ProcessInfo> processInfos = new ArrayList<>(processCount);
for (int threadIdx = 0; threadIdx < batch.size(); threadIdx++) {
List<Integer> balancedPageNumbersSubList = batch.get(threadIdx);
processInfos.add(new ProcessInfo(threadIdx, balancedPageNumbersSubList));
}
processInfoBatches.add(processInfos);
}
return processInfoBatches;
}
@SneakyThrows
private ProcessCmdsAndRenderedImageFiles buildCmdArgs(Integer processIdx,
Integer batchIdx,
List<Integer> stitchedImagePageIndices,
Path outputDir,
String documentAbsolutePath) {
String imagePathFormat = outputDir.resolve("output_" + processIdx + "_" + batchIdx + ".%04d" + FORMAT).toFile().toString();
Map<Integer, RenderedPageImageFile> fullPageImages = new HashMap<>();
for (int i = 0; i < stitchedImagePageIndices.size(); i++) {
Integer pageNumber = stitchedImagePageIndices.get(i);
fullPageImages.put(pageNumber, new RenderedPageImageFile(pageNumber, String.format(imagePathFormat, i + 1)));
}
String[] cmdArgs = buildCmdArgs(stitchedImagePageIndices, documentAbsolutePath, imagePathFormat);
return new ProcessCmdsAndRenderedImageFiles(cmdArgs, fullPageImages);
}
private String[] buildCmdArgs(List<Integer> stitchedImagePageIndices, String documentAbsolutePath, String imagePathFormat) {
StringBuilder sPageList = new StringBuilder();
int i = 1;
for (Integer integer : stitchedImagePageIndices) {
sPageList.append(integer);
if (i < stitchedImagePageIndices.size()) {
sPageList.append(",");
}
i++;
}
String[] cmdArgs = new String[]{"gs", "-dNOPAUSE", "-sDEVICE=" + DEVICE, "-r" + settings.getDpi(), "-sPageList=" + sPageList, "-sOutputFile=" + imagePathFormat, documentAbsolutePath, "-c", "quit"};
return cmdArgs;
}
@SneakyThrows
private Process executeProcess(ProcessCmdsAndRenderedImageFiles processInfo, BlockingQueue<RenderedPageImageFile> imageFileCollectorQueue) {
Process p = Runtime.getRuntime().exec(processInfo.cmdArgs());
InputStream stdOut = p.getInputStream();
GhostScriptOutputHandler stdOutLogger = GhostScriptOutputHandler.stdOut(stdOut, processInfo.renderedPageImageFiles(), imageFileCollectorQueue);
InputStream stdError = p.getErrorStream();
GhostScriptOutputHandler stdErrorLogger = GhostScriptOutputHandler.errorHandler(stdError);
stdOutLogger.start();
stdErrorLogger.start();
return p;
}
private record ProcessCmdsAndRenderedImageFiles(String[] cmdArgs, Map<Integer, RenderedPageImageFile> renderedPageImageFiles) {
}
private record ProcessInfo(Integer processIdx, List<Integer> stitchedPageNumbers) {
}
}
@@ -0,0 +1,24 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.io.FileInputStream;
import java.util.List;
import io.github.karols.hocr4j.Page;
import lombok.SneakyThrows;
import lombok.experimental.UtilityClass;
@UtilityClass
public class HOcrPageParser {
@SneakyThrows
public Page extractHocrPage(String tesseractOutputFileName) {
String hOcrString;
try (var hocrIn = new FileInputStream(tesseractOutputFileName + ".hocr")) {
hOcrString = new String(hocrIn.readAllBytes());
}
return Page.fromHocr(List.of(hOcrString)).get(0);
}
}
@@ -0,0 +1,13 @@
package com.knecon.fforesight.service.ocr.processor.service;
import org.springframework.stereotype.Service;
@Service
public interface IOcrMessageSender {
void sendUpdate(String fileId, int finishedImages, int totalImages);
void sendOcrFinished(String fileId, int totalImages);
}
@@ -0,0 +1,96 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.awt.geom.Rectangle2D;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import org.apache.pdfbox.contentstream.PDFStreamEngine;
import org.apache.pdfbox.contentstream.operator.DrawObject;
import org.apache.pdfbox.contentstream.operator.Operator;
import org.apache.pdfbox.contentstream.operator.state.Concatenate;
import org.apache.pdfbox.contentstream.operator.state.Restore;
import org.apache.pdfbox.contentstream.operator.state.Save;
import org.apache.pdfbox.contentstream.operator.state.SetGraphicsStateParameters;
import org.apache.pdfbox.contentstream.operator.state.SetMatrix;
import org.apache.pdfbox.cos.COSBase;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.graphics.PDXObject;
import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject;
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
import org.apache.pdfbox.util.Matrix;
import com.knecon.fforesight.service.ocr.processor.model.ExtractedImage;
import com.knecon.fforesight.service.ocr.processor.model.QuadPoint;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import lombok.Getter;
import lombok.SneakyThrows;
@Getter
public class ImageStreamEngine extends PDFStreamEngine {
private List<ExtractedImage> imagesOnCurrentPage;
private OcrServiceSettings settings;
private int pageNum;
public ImageStreamEngine(OcrServiceSettings settings) {
this.settings = settings;
// preparing PDFStreamEngine
addOperator(new Concatenate(this));
addOperator(new DrawObject(this));
addOperator(new SetGraphicsStateParameters(this));
addOperator(new Save(this));
addOperator(new Restore(this));
addOperator(new SetMatrix(this));
}
@Override
protected void processOperator(Operator operator, List<COSBase> operands) throws IOException {
String operation = operator.getName();
if ("Do".equals(operation)) {
COSName objectName = (COSName) operands.get(0);
// get the PDF object
PDXObject xobject = getResources().getXObject(objectName);
// check if the object is an image object
if (xobject instanceof PDImageXObject imageXObject) {
if (imageXObject.getWidth() < settings.getMinImageWidth() || imageXObject.getHeight() < settings.getMinImageHeight()) {
return;
}
Matrix imageCTM = getGraphicsState().getCurrentTransformationMatrix();
this.imagesOnCurrentPage.add(new ExtractedImage(pageNum,
QuadPoint.fromRectangle2D(new Rectangle2D.Double(0, 0, imageXObject.getWidth(), imageXObject.getHeight())),
imageXObject.getHeight(),
imageXObject.getWidth(),
imageXObject.getImage(),
imageCTM,
imagesOnCurrentPage.size(),
imageXObject.getColorSpace()));
} else if (xobject instanceof PDFormXObject) {
PDFormXObject form = (PDFormXObject) xobject;
showForm(form);
}
} else {
super.processOperator(operator, operands);
}
}
@SneakyThrows
public void processPage(int pageNum, PDPage page) {
this.pageNum = pageNum;
this.imagesOnCurrentPage = new LinkedList<>();
super.processPage(page);
}
}
@@ -0,0 +1,161 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.stream.IntStream;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import org.springframework.util.FileSystemUtils;
import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
import com.knecon.fforesight.service.ocr.processor.model.OcrResultToWrite;
import com.knecon.fforesight.service.ocr.processor.model.OcrImage;
import com.knecon.fforesight.service.ocr.processor.model.OcrResult;
import com.knecon.fforesight.service.ocr.processor.service.scriptdetection.FontStyleDetector;
import com.knecon.fforesight.service.ocr.processor.service.threads.OCRThread;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class OCRService {
FileStorageService fileStorageService;
OcrServiceSettings settings;
IOcrMessageSender ocrMessageSender;
WatermarkRemovalService watermarkRemovalService;
InvisibleElementRemovalService invisibleElementRemovalService;
OcrResultWriter ocrResultWriter;
GhostScriptService ghostScriptService;
FontStyleDetector boldDetector;
/**
* Starts the OCR-Process: Collecting images (via threads),
* looking for stitchedImages (if so converting the current page to an image with ghostscript and work on this instead),
* perform tesseract-ocr on these images (via threads) and write the generated ocr-text as invisible elements.
*
* @param dossierId Id of dossier
* @param fileId Id of file
* @param out OutputStream where to write to
*/
@SneakyThrows
public void runOcrOnDocument(String dossierId, String fileId, OutputStream out) {
try (InputStream fileStream = removeWatermarkIfEnabled(dossierId, fileId); ByteArrayOutputStream transferOutputStream = new ByteArrayOutputStream()) {
invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
try (InputStream transferInputStream = new ByteArrayInputStream(transferOutputStream.toByteArray())) {
log.info("Starting OCR for file {}", fileId);
long ocrStart = System.currentTimeMillis();
Statistics stats = runOcr(transferInputStream, out, fileId, dossierId);
long ocrEnd = System.currentTimeMillis();
log.info("ocr successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, String.format("%.1f", (ocrEnd - ocrStart) / 1000.0));
log.info("Runtime breakdown: {}", stats);
}
}
}
private InputStream removeWatermarkIfEnabled(String dossierId, String fileId) throws IOException {
if (settings.isRemoveWatermark()) {
try (var in = fileStorageService.getOriginalFileAsStream(dossierId, fileId); var transferOutputStream = new ByteArrayOutputStream()) {
watermarkRemovalService.removeWatermarks(in, transferOutputStream);
return new ByteArrayInputStream(transferOutputStream.toByteArray());
}
}
return fileStorageService.getOriginalFileAsStream(dossierId, fileId);
}
@SneakyThrows
public Statistics runOcr(InputStream in, OutputStream out, String fileId, String dossierId) {
long timestamp;
Path tmpDir = Path.of(OsUtils.getTemporaryDirectory()).resolve(dossierId + "-" + fileId);
Path tmpImageDir = tmpDir.resolve("images");
Path tesseractOutputDir = tmpDir.resolve("tesseract_output");
tesseractOutputDir.toFile().mkdirs();
tmpImageDir.toFile().mkdirs();
File documentFile = OsUtils.writeFileToTmpFolder(in, tmpDir);
Statistics stats;
try (PDDocument document = Loader.loadPDF(documentFile)) {
OcrProgressLogger logger = new OcrProgressLogger(document.getNumberOfPages(), ocrMessageSender, fileId);
int numberOfExtractThreads = Math.min(settings.getImageExtractThreadCount(), document.getNumberOfPages());
int numberOfOcrThreads = Math.min(settings.getOcrThreadCount(), document.getNumberOfPages());
stats = new Statistics(numberOfExtractThreads, numberOfOcrThreads);
BlockingQueue<OcrImage> ocrImageQueue = new ArrayBlockingQueue<>((int) (1.5 * numberOfOcrThreads));
OcrImageFactory ocrImageFactory = new OcrImageFactory(document,
documentFile,
tmpImageDir,
numberOfExtractThreads,
ghostScriptService,
ocrImageQueue,
logger,
settings,
stats);
ocrImageFactory.start();
List<OcrResult> ocrResults = new LinkedList<>();
List<OCRThread> ocrThreads = IntStream.range(0, numberOfOcrThreads)
.boxed()
.map(id -> new OCRThread(id, ocrImageQueue, tesseractOutputDir, ocrResults, logger, stats, settings))
.peek(Thread::start)
.toList();
log.info("Started {} OCR consumer threads, listening for images on the queue", ocrThreads.size());
ocrImageFactory.join();
log.info("Processed all images, interrupting ocr threads");
ocrThreads.forEach(Thread::interrupt);
for (OCRThread ocrThread : ocrThreads) {
ocrThread.join();
}
log.info("Tesseract OCR has finished for file {} and dossier {}", fileId, dossierId);
timestamp = System.currentTimeMillis();
Map<Integer, List<OcrResultToWrite>> imageWithTextPositionsPerPage = boldDetector.detectBold(ocrResults, document);
stats.increaseFontStyleDetectionDuration(System.currentTimeMillis() - timestamp);
timestamp = System.currentTimeMillis();
var dictionariesToUpdate = ocrResultWriter.drawOcrResultsToPdf(document, imageWithTextPositionsPerPage);
log.info("Saving document");
document.saveIncremental(out, dictionariesToUpdate);
stats.increaseWritingTextDuration(System.currentTimeMillis() - timestamp);
FileSystemUtils.deleteRecursively(tmpDir);
logger.sendFinished();
return stats;
}
}
}
@@ -0,0 +1,105 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.io.File;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.stream.Collectors;
import org.apache.pdfbox.pdmodel.PDDocument;
import com.knecon.fforesight.service.ocr.processor.model.ExtractedImage;
import com.knecon.fforesight.service.ocr.processor.model.OcrImage;
import com.knecon.fforesight.service.ocr.processor.model.UnprocessedImage;
import com.knecon.fforesight.service.ocr.processor.service.threads.ImageExtractionThread;
import com.knecon.fforesight.service.ocr.processor.service.threads.ImageProcessingThread;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.utils.ListSplittingUtils;
import lombok.AccessLevel;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class OcrImageFactory {
PDDocument document;
File documentFile;
Path tmpImageDir;
GhostScriptService ghostScriptService;
BlockingQueue<UnprocessedImage> imageProcessingQueue;
ImageProcessingThread imageProcessingThread;
BlockingQueue<OcrImage> imageOutputQueue;
List<ImageExtractionThread> imageExtractionThreads;
List<Integer> stitchedPageNumbers;
Statistics stats;
public OcrImageFactory(PDDocument document,
File documentFile,
Path tmpImageDir,
int numberOfThreads,
GhostScriptService ghostScriptService,
BlockingQueue<OcrImage> imageOcrQueue,
OcrProgressLogger logger,
OcrServiceSettings settings,
Statistics stats) {
this.document = document;
this.documentFile = documentFile;
this.tmpImageDir = tmpImageDir;
this.ghostScriptService = ghostScriptService;
this.imageOutputQueue = imageOcrQueue;
this.imageProcessingQueue = new ArrayBlockingQueue<>(imageOcrQueue.remainingCapacity());
this.stitchedPageNumbers = Collections.synchronizedList(new LinkedList<>());
this.stats = stats;
this.imageExtractionThreads = new ArrayList<>(numberOfThreads);
List<List<Integer>> balancedPageNumbers = ListSplittingUtils.buildBalancedContinuousSublist(document.getNumberOfPages(), numberOfThreads);
for (int i = 0; i < balancedPageNumbers.size(); i++) {
imageExtractionThreads.add(new ImageExtractionThread(i, balancedPageNumbers.get(i), documentFile, logger, stats, settings, imageProcessingQueue, stitchedPageNumbers));
}
this.imageProcessingThread = new ImageProcessingThread(imageProcessingQueue, imageOcrQueue, stats, settings, document);
log.info("Started {} image extraction threads, with ({}) pages each",
imageExtractionThreads.size(),
imageExtractionThreads.stream().map(ImageExtractionThread::getPageIndices).map(List::size).map(String::valueOf).collect(Collectors.joining(", ")));
}
public void start() {
for (ImageExtractionThread imageExtractionThread : imageExtractionThreads) {
imageExtractionThread.start();
}
imageProcessingThread.start();
}
@SneakyThrows
public void join() {
for (ImageExtractionThread imageExtractionThread : imageExtractionThreads) {
imageExtractionThread.join();
}
if (!stitchedPageNumbers.isEmpty()) {
ghostScriptService.renderPagesAsImagesBatchedAndAddToQueue(stitchedPageNumbers, documentFile.toString(), tmpImageDir, document, imageProcessingQueue, stats);
}
imageProcessingThread.setAllImagesExtracted(true);
imageProcessingThread.interrupt();
imageProcessingThread.join();
}
}
@@ -0,0 +1,91 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import com.knecon.fforesight.service.ocr.processor.model.ExtractedOcrImage;
import com.knecon.fforesight.service.ocr.processor.model.OcrImage;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class OcrProgressLogger {
Set<ImageNumberWithPageNumber> imagesToProcess;
Set<ImageNumberWithPageNumber> processedImages;
IOcrMessageSender ocrMessageSender;
String fileId;
public OcrProgressLogger(int totalPageCount, IOcrMessageSender ocrMessageSender, String fileId) {
this.ocrMessageSender = ocrMessageSender;
this.fileId = fileId;
this.imagesToProcess = Collections.synchronizedSet(new HashSet<>(totalPageCount));
for (int i = 0; i < totalPageCount; i++) {
imagesToProcess.add(new ImageNumberWithPageNumber(0, i + 1));
}
this.processedImages = Collections.synchronizedSet(new HashSet<>(totalPageCount));
}
public void logImageFinished(OcrImage image, int psm) {
this.processedImages.add(new ImageNumberWithPageNumber(image.getNumberOnPage(), image.getPageNumber()));
if (image instanceof ExtractedOcrImage) {
log.info("{}/{}: Finished image {} on page {} with rotation {}, used PSM {}, quad-point: {}",
processedImages.size(),
imagesToProcess.size(),
image.getNumberOnPage(),
image.getPageNumber(),
image.getRotationDegrees(),
psm,
image.getImageCoordinatesInInitialUserSpace());
} else {
log.info("{}/{}: Finished page {} as fully rendered page with rotation {}, used PSM {}",
processedImages.size(),
imagesToProcess.size(),
image.getPageNumber(),
image.getRotationDegrees(),
psm);
}
ocrMessageSender.sendUpdate(fileId, this.processedImages.size(), this.imagesToProcess.size());
}
public void logPageSkipped(Integer pageIndex) {
var pageDummy = new ImageNumberWithPageNumber(0, pageIndex);
this.imagesToProcess.remove(pageDummy);
log.debug("{}/{}: No images to ocr on page {}", processedImages.size(), imagesToProcess.size(), pageIndex);
ocrMessageSender.sendUpdate(fileId, this.processedImages.size(), imagesToProcess.size());
}
public void addImagesToProcess(int pageNumber, int imageNumber) {
this.imagesToProcess.add(new ImageNumberWithPageNumber(imageNumber, pageNumber));
}
public void sendFinished() {
log.info("{}/{}: Finished OCR on all images", processedImages.size(), imagesToProcess.size());
ocrMessageSender.sendOcrFinished(fileId, imagesToProcess.size());
}
private record ImageNumberWithPageNumber(int imageNumber, int pageNumber) {
}
}
@@ -0,0 +1,252 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.awt.Color;
import java.awt.geom.Point2D;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentGroup;
import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
import org.apache.pdfbox.pdmodel.graphics.state.RenderingMode;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.ocr.processor.model.OcrResultToWrite;
import com.knecon.fforesight.service.ocr.processor.model.QuadPoint;
import com.knecon.fforesight.service.ocr.processor.model.TextPositionInImage;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class OcrResultWriter {
static String ocrLayerName = "knecon OCR";
OcrServiceSettings settings;
@SneakyThrows
public Set<COSDictionary> drawOcrResultsToPdf(PDDocument document, Map<Integer, List<OcrResultToWrite>> imagesWithResultsPerPage) {
Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
imagesWithResultsPerPage.keySet().forEach(pageNumber -> drawResultsPerPage(document, pageNumber, imagesWithResultsPerPage.get(pageNumber), dictionariesToUpdate));
dictionariesToUpdate.add(document.getDocumentInformation().getCOSObject());
return dictionariesToUpdate;
}
@SneakyThrows
private void drawResultsPerPage(PDDocument document, Integer pageNumber, List<OcrResultToWrite> ocrResultToWrite, Set<COSDictionary> dictionariesToUpdate) {
var pdPage = document.getPage(pageNumber - 1);
PDOptionalContentGroup textDebugLayer = new PDOptionalContentGroup(ocrLayerName);
PDOptionalContentGroup bBoxDebugLayer = new PDOptionalContentGroup(ocrLayerName + "BBox");
if (settings.isDebug()) {
textDebugLayer = addOptionalGroup(ocrLayerName, document, pdPage, dictionariesToUpdate);
bBoxDebugLayer = addOptionalGroup(ocrLayerName + " BBox", document, pdPage, dictionariesToUpdate);
}
escapeContentStreams(document, pdPage);
List<TextPositionInImage> words = ocrResultToWrite.stream().map(OcrResultToWrite::textPositionInImage).flatMap(Collection::stream).toList();
try (var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true)) {
// write invisible ocr text inside tagged content
contentStream.beginMarkedContent(settings.getOcrMarkedContentTag());
contentStream.saveGraphicsState();
contentStream.setNonStrokingColor(Color.BLUE);
contentStream.setStrokingColor(Color.BLUE);
contentStream.setLineWidth(1);
words.forEach(word -> drawInvisibleWord(word, contentStream));
contentStream.restoreGraphicsState();
contentStream.endMarkedContent();
if (settings.isDebug()) { // must not be written, as it will interfere with layout parsing
// write visible ocr text inside optional group
contentStream.beginMarkedContent(COSName.OC, textDebugLayer);
contentStream.saveGraphicsState();
words.forEach(word -> drawVisibleWord(word, contentStream));
contentStream.restoreGraphicsState();
contentStream.endMarkedContent();
// write word bounding boxes (tesseract output) inside optional group
contentStream.beginMarkedContent(COSName.OC, bBoxDebugLayer);
contentStream.saveGraphicsState();
ocrResultToWrite.stream()
.map(OcrResultToWrite::imageBoundingBox)
.forEach(imagePosition -> drawGrid(contentStream, imagePosition));
words.stream().map(TextPositionInImage::getTransformedTextBBox).forEach(word -> drawRectangle(contentStream, word));
contentStream.restoreGraphicsState();
contentStream.endMarkedContent();
}
}
dictionariesToUpdate.add(pdPage.getCOSObject());
dictionariesToUpdate.add(pdPage.getResources().getCOSObject());
}
@SneakyThrows
private static void escapeContentStreams(PDDocument document, PDPage pdPage) {
// We need to append to the contentstream, otherwise the content could be overlapped by images
// But we also need to save the graphics state before, such that our appended content cannot be affected by previous contentstreams with side-effects, such as not escaped matrix transformations
try (var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.PREPEND, false)) {
contentStream.saveGraphicsState();
}
try (var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, false)) {
contentStream.restoreGraphicsState();
}
}
private PDOptionalContentGroup addOptionalGroup(String ocrLayerName, PDDocument document, PDPage pdPage, Set<COSDictionary> dictionariesToUpdate) {
PDDocumentCatalog catalog = document.getDocumentCatalog();
PDOptionalContentProperties ocprops = catalog.getOCProperties();
if (ocprops == null) {
ocprops = new PDOptionalContentProperties();
catalog.setOCProperties(ocprops);
}
PDOptionalContentGroup layer = null;
if (ocprops.hasGroup(ocrLayerName)) {
layer = ocprops.getGroup(ocrLayerName);
} else {
layer = new PDOptionalContentGroup(ocrLayerName);
ocprops.addGroup(layer);
}
// enable debug layers by default only when DEBUG flag is set.
ocprops.setGroupEnabled(layer, settings.isDebug());
PDResources resources = pdPage.getResources();
if (resources == null) {
resources = new PDResources();
pdPage.setResources(resources);
}
dictionariesToUpdate.add(catalog.getCOSObject());
return layer;
}
@SneakyThrows
private void drawRectangle(PDPageContentStream contentStream, QuadPoint rect) {
contentStream.saveGraphicsState();
contentStream.setLineWidth(1);
contentStream.moveTo((float) rect.a().getX(), (float) rect.a().getY());
contentStream.lineTo((float) rect.b().getX(), (float) rect.b().getY());
contentStream.setStrokingColor(Color.ORANGE);
contentStream.stroke();
contentStream.moveTo((float) rect.b().getX(), (float) rect.b().getY());
contentStream.lineTo((float) rect.c().getX(), (float) rect.c().getY());
contentStream.setStrokingColor(Color.BLUE);
contentStream.stroke();
contentStream.moveTo((float) rect.c().getX(), (float) rect.c().getY());
contentStream.lineTo((float) rect.d().getX(), (float) rect.d().getY());
contentStream.setStrokingColor(Color.GREEN);
contentStream.stroke();
contentStream.moveTo((float) rect.d().getX(), (float) rect.d().getY());
contentStream.lineTo((float) rect.a().getX(), (float) rect.a().getY());
contentStream.setStrokingColor(Color.MAGENTA);
contentStream.stroke();
contentStream.restoreGraphicsState();
}
private void drawInvisibleWord(TextPositionInImage word, PDPageContentStream contentStream) {
drawWord(word, contentStream, RenderingMode.NEITHER);
}
private void drawVisibleWord(TextPositionInImage word, PDPageContentStream contentStream) {
drawWord(word, contentStream, RenderingMode.FILL);
}
// @SneakyThrows
private void drawWord(TextPositionInImage position, PDPageContentStream contentStream, RenderingMode renderingMode) {
try {
contentStream.setNonStrokingColor(switch (position.getFontStyle()) {
case BOLD -> Color.RED;
case ITALIC -> Color.GREEN;
default -> Color.BLUE;
});
contentStream.beginText();
contentStream.setRenderingMode(renderingMode);
contentStream.setFont(position.getFont(), (float) position.getFontSize());
contentStream.setTextMatrix(position.getTextMatrix());
contentStream.showText(position.getText());
contentStream.endText();
} catch (Exception e) {
log.error("Failed to write text {}", position.getText());
log.error(e.getMessage());
}
}
@SneakyThrows
private void drawGrid(PDPageContentStream contentStream, QuadPoint rect) {
drawRectangle(contentStream, rect);
contentStream.saveGraphicsState();
contentStream.setStrokingColor(Color.BLACK);
contentStream.setLineWidth(0.2F);
int nRows = 8;
int nCols = 8;
Point2D abStep = new Point2D.Double((rect.b().getX() - rect.a().getX()) / (nRows + 1), (rect.b().getY() - rect.a().getY()) / (nRows + 1));
Point2D start = add(rect.a(), abStep);
Point2D end = add(rect.d(), abStep);
for (int row = 0; row < nRows; ++row) {
drawLine(start, end, contentStream);
start = add(start, abStep);
end = add(end, abStep);
}
Point2D adStep = new Point2D.Double((rect.d().getX() - rect.a().getX()) / (nCols + 1), (rect.d().getY() - rect.a().getY()) / (nCols + 1));
start = add(rect.a(), adStep);
end = add(rect.b(), adStep);
for (int col = 0; col < nCols; ++col) {
drawLine(start, end, contentStream);
start = add(start, adStep);
end = add(end, adStep);
}
contentStream.restoreGraphicsState();
}
@SneakyThrows
private void drawLine(Point2D a, Point2D b, PDPageContentStream contentStream) {
contentStream.moveTo((float) a.getX(), (float) a.getY());
contentStream.lineTo((float) b.getX(), (float) b.getY());
contentStream.stroke();
}
private Point2D add(Point2D a, Point2D b) {
return new Point2D.Double(a.getX() + b.getX(), a.getY() + b.getY());
}
}
@@ -1,4 +1,10 @@
package com.iqser.red.service.ocr.v1.server.utils;
package com.knecon.fforesight.service.ocr.processor.service;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import org.apache.commons.lang3.StringUtils;
@@ -21,6 +27,14 @@ public final class OsUtils {
return addBackSlashAtEnd(getTemporaryDirectory()) + addBackSlashAtEnd(SERVICE_NAME) + addBackSlashAtEnd(suffix) + addBackSlashAtEnd(fileId);
}
public static File writeFileToTmpFolder(InputStream in, Path tmpDir) throws IOException {
File pdfFile = tmpDir.resolve("document.pdf").toFile();
try (var fileOut = new FileOutputStream(pdfFile)) {
fileOut.write(in.readAllBytes());
}
return pdfFile;
}
private static boolean isWindows() {
@@ -55,4 +69,10 @@ public final class OsUtils {
return "/tmp";
}
public static String createTmpFileName(String filename, String suffix) {
return Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(filename).getFileName()).toString().replace(".pdf", "_" + suffix + ".pdf");
}
}
@@ -0,0 +1,85 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class Statistics {
List<Long> imageExtraction;
List<Long> tesseractDuration;
AtomicLong pdf2ImgDuration;
AtomicLong writingTextDuration;
AtomicLong imageProcessingDuration;
AtomicLong fontStyleDetectionDuration;
public Statistics(int numberOfExtractThreads, int numberOfOcrThreads) {
this.imageExtraction = Collections.synchronizedList(new ArrayList<>(Collections.nCopies(numberOfExtractThreads, 0L)));
this.tesseractDuration = Collections.synchronizedList(new ArrayList<>(Collections.nCopies(numberOfOcrThreads, 0L)));
this.fontStyleDetectionDuration = new AtomicLong(0);
this.pdf2ImgDuration = new AtomicLong(0);
this.writingTextDuration = new AtomicLong(0);
this.imageProcessingDuration = new AtomicLong(0);
}
public void increaseImageExtraction(int threadId, long duration) {
imageExtraction.set(threadId, imageExtraction.get(threadId) + duration);
}
public void increaseImageProcessing(long duration) {
imageProcessingDuration.addAndGet(duration);
}
public void increaseTesseractDuration(int threadId, long duration) {
tesseractDuration.set(threadId, tesseractDuration.get(threadId) + duration);
}
public void increasePDF2ImgDuration(long duration) {
pdf2ImgDuration.addAndGet(duration);
}
public void increaseWritingTextDuration(long duration) {
writingTextDuration.addAndGet(duration);
}
public void increaseFontStyleDetectionDuration(long duration) {
fontStyleDetectionDuration.addAndGet(duration);
}
@Override
public String toString() {
return String.format(
"imageExtraction: mean %.2f s, max %.2f s, min %.2f, tesseract: mean %.2f s, max %.2f s, min %.2f, ImageProcessing=%.2f s, PDF2Img=%.2f s, writingText=%.2f s, FontstyleDetection=%.2f s",
((float) imageExtraction.stream().mapToLong(Long::longValue).average().orElse(0) / 1000),
((float) imageExtraction.stream().mapToLong(Long::longValue).max().orElse(0) / 1000),
((float) imageExtraction.stream().mapToLong(Long::longValue).min().orElse(0) / 1000),
((float) tesseractDuration.stream().mapToLong(Long::longValue).average().orElse(0) / 1000),
((float) tesseractDuration.stream().mapToLong(Long::longValue).max().orElse(0) / 1000),
((float) tesseractDuration.stream().mapToLong(Long::longValue).min().orElse(0) / 1000),
(float) imageProcessingDuration.get() / 1000,
(float) pdf2ImgDuration.get() / 1000,
(float) writingTextDuration.get() / 1000,
(float) fontStyleDetectionDuration.get() / 1000);
}
}
@@ -0,0 +1,42 @@
package com.knecon.fforesight.service.ocr.processor.service.fonts;
import org.apache.pdfbox.pdmodel.font.PDFont;
import com.knecon.fforesight.service.ocr.processor.model.FontMetrics;
import com.knecon.fforesight.service.ocr.processor.model.HeightAndDescent;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
public interface FontMetricsFactory {
default FontMetrics calculateMetrics(String text, double textWidth, double textHeight) {
HeightAndDescent heightAndDescent = calculateHeightAndDescent(text);
float fontSize = calculateFontSize(text, textWidth);
float heightScaling = (float) ((textHeight / (heightAndDescent.height() - heightAndDescent.descent())) * 1000) / fontSize;
return new FontMetrics((heightAndDescent.descent() / 1000) * fontSize, fontSize, heightScaling);
}
@SneakyThrows
default float calculateFontSize(String text, double textWidth) {
float width;
try {
width = getFont().getStringWidth(text);
} catch (IllegalArgumentException e) {
// this means, the font has no glyph for this character
width = getFont().getAverageFontWidth() * text.length();
}
return (float) (textWidth / width) * 1000;
}
PDFont getFont();
HeightAndDescent calculateHeightAndDescent(String text);
}
@@ -0,0 +1,5 @@
package com.knecon.fforesight.service.ocr.processor.service.fonts;
public enum FontStyle {
REGULAR, BOLD, ITALIC
}
@@ -0,0 +1,100 @@
package com.knecon.fforesight.service.ocr.processor.service.fonts;
import java.io.ByteArrayInputStream;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.apache.fontbox.ttf.GlyphData;
import org.apache.fontbox.ttf.TTFParser;
import org.apache.fontbox.ttf.TrueTypeFont;
import org.apache.pdfbox.io.RandomAccessReadBuffer;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.pdmodel.font.PDType0Font;
import com.knecon.fforesight.service.ocr.processor.model.HeightAndDescent;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import software.amazon.awssdk.services.s3.endpoints.internal.Value;
@Slf4j
@RequiredArgsConstructor
public class Type0FontMetricsFactory implements FontMetricsFactory {
private final PDType0Font type0Font;
private final TrueTypeFont trueTypeFont;
// for this specific font back-/forward-slashes have a lot of descent screwing up the font size and therefore bold detection. So if we find such a character we ignore its descent.
private static final Set<Integer> slashGlyphIds = Set.of(18, 63);
public static Type0FontMetricsFactory regular(PDDocument document) {
return createFromResource("fonts/cmu-regular.ttf", document);
}
public static Type0FontMetricsFactory bold(PDDocument document) {
return createFromResource("fonts/cmu-bold.ttf", document);
}
@SneakyThrows
private static Type0FontMetricsFactory createFromResource(String resourcePath, PDDocument document) {
try (var in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourcePath); var buffer = new RandomAccessReadBuffer(in)) {
TrueTypeFont trueTypeFont = new TTFParser().parse(buffer); // since Type0Font can be descendant from any font, we need to remember the original TrueTypeFont for the glyph information
PDType0Font type0Font = PDType0Font.load(document, trueTypeFont, true); // use Type0Font for unicode support
return new Type0FontMetricsFactory(type0Font, trueTypeFont);
}
}
@SneakyThrows
public HeightAndDescent calculateHeightAndDescent(String text) {
byte[] bytes;
try {
bytes = type0Font.encode(text);
} catch (IllegalArgumentException e) {
log.warn("The string {} could not be parsed, using average height and descent", text);
return new HeightAndDescent(800, -50);
}
ByteArrayInputStream in = new ByteArrayInputStream(bytes);
float descent = 0;
float height = 0;
while (in.available() > 0) {
try {
int code = type0Font.readCode(in);
int glyphId = type0Font.codeToGID(code);
GlyphData glyph = trueTypeFont.getGlyph().getGlyph(glyphId);
if (glyph == null || glyph.getBoundingBox() == null) {
continue;
}
if (!slashGlyphIds.contains(glyphId)) {
descent = Math.min(descent, glyph.getYMinimum());
}
height = Math.max(height, glyph.getYMaximum());
} catch (Exception e) {
log.warn("descent and height of string {} could not be parsed, using average fallback value!", text);
}
}
// some characters like comma or minus return very small height values, while tesseract still returns a normal-sized bounding box and therefore exploding the height scaling factors,
// so we need a minimum value. Here, 500 seems optimal for the characters "-", ",", "_"
return new HeightAndDescent(Math.max(height, 500), descent);
}
@Override
public PDFont getFont() {
return type0Font;
}
}
@@ -0,0 +1,158 @@
package com.knecon.fforesight.service.ocr.processor.service.scriptdetection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
import org.apache.commons.math3.ml.clustering.Cluster;
import org.apache.commons.math3.ml.clustering.DBSCANClusterer;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.ocr.processor.model.OcrResult;
import com.knecon.fforesight.service.ocr.processor.model.OcrResultToWrite;
import com.knecon.fforesight.service.ocr.processor.model.scriptdetection.FontStyleDetectionModel;
import com.knecon.fforesight.service.ocr.processor.model.scriptdetection.TextPositionAndWordImage;
import com.knecon.fforesight.service.ocr.processor.model.scriptdetection.WordImage;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontMetricsFactory;
import com.knecon.fforesight.service.ocr.processor.service.fonts.FontStyle;
import com.knecon.fforesight.service.ocr.processor.service.fonts.Type0FontMetricsFactory;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class FontStyleDetector {
OcrServiceSettings settings;
StrokeWidthCalculator strokeWidthCalculator;
/**
* Implementation of the MOBDoB algorithm, refer to the paper here:
* <a href="http://mile.ee.iisc.ac.in/publications/softCopy/DocumentAnalysis/Sai_NCVPRIPG2013.pdf">Script Independent Detection of Bold Words in Multi Font-size Documents</a>
* <p>
* As a high level overview: We cluster all text based on its font size. We determine the cluster with the most words. This is assumed to be regular text.
* We then estimate the average stroke width of that cluster by thinning all text to a single pixel and calculating the ratio of remaining pixels.
* (<a href="http://www.leptonica.org/papers/conn.pdf">Leptonica Documentation on thinning</a>)
* For each word we scale this average strokewidth based on its fontsize compared to the most common fontsize.
* Using the scaled strokewidth we do an opening operation.
* (<a href="https://en.wikipedia.org/wiki/Opening_(morphology)">Opening (Morphology)</a>).
* We then threshold the ratio of remaining pixels to determine whether a word is bold or not.
* <p>
* I did take some liberties though. Firstly, the paper uses text height without ascender/descender height for the clustering. I'm using the previously implemented font size.
* But this is based on text width. Thus, I'm also using the height scaling factor to scale the font size by the text height.
* The paper does not describe its clustering algorithm, so I've decided on DBSCAN due to its good runtime and readily available implementation by apache commons math.
* Moreover, the paper states that stroke width scales linearly with text height. I've come to the conclusion this is not the case.
* It seems it scales with the square root of the text height. Or at least this seemed to give the best results.
*/
public Map<Integer, List<OcrResultToWrite>> detectBold(List<OcrResult> ocrResults, PDDocument document) {
FontMetricsFactory fontMetricsFactory = Type0FontMetricsFactory.regular(document);
if (!settings.isBoldDetection()) {
return OcrResultToWrite.buildOcrResultsToWrite(ocrResults, fontMetricsFactory);
}
Map<Integer, List<OcrResultToWrite>> ocrResultToWritePerPage = new HashMap<>();
DBSCANClusterer<TextPositionAndWordImage> clusterer = new DBSCANClusterer<>(0.5, 1);
FontMetricsFactory boldFontMetricsFactory = Type0FontMetricsFactory.bold(document);
for (OcrResult result : ocrResults) {
FontStyleDetectionModel fontStyleDetectionModel = FontStyleDetectionModel.fromOcrResult(result, fontMetricsFactory, settings);
List<Cluster<TextPositionAndWordImage>> clusters = clusterer.cluster(fontStyleDetectionModel.getTextPositionsAndWordImages());
Optional<Cluster<TextPositionAndWordImage>> largestCluster = clusters.stream().max(Comparator.comparingInt(cluster -> cluster.getPoints().size()));
if (largestCluster.isEmpty()) {
insertResultIntoMap(result.image().getPageNumber(), ocrResultToWritePerPage, fontStyleDetectionModel);
continue;
}
List<TextPositionAndWordImage> wordsWithMostCommonTextHeight = largestCluster.get().getPoints();
double standardTextHeight = calculateStandardTextheight(wordsWithMostCommonTextHeight);
double regularStrokeWidth = calculateRegularStrokeWidth(wordsWithMostCommonTextHeight);
for (TextPositionAndWordImage textPositionsAndWordImage : fontStyleDetectionModel.getTextPositionsAndWordImages()) {
decideOnFontStyle(textPositionsAndWordImage, regularStrokeWidth, standardTextHeight, boldFontMetricsFactory);
}
insertResultIntoMap(result.image().getPageNumber(), ocrResultToWritePerPage, fontStyleDetectionModel);
fontStyleDetectionModel.dispose();
}
log.info("Finished bold detection");
return ocrResultToWritePerPage;
}
private static double calculateStandardTextheight(List<TextPositionAndWordImage> wordsWithMostCommonTextHeight) {
return wordsWithMostCommonTextHeight.stream()
.map(TextPositionAndWordImage::getWordImage)
.mapToDouble(WordImage::getTextHeight)
.filter(Double::isFinite)
.average()
.orElseThrow();
}
private double calculateRegularStrokeWidth(List<TextPositionAndWordImage> wordsWithMostCommonTextHeight) {
return wordsWithMostCommonTextHeight.stream()
.mapToDouble(textPositionAndWordImage -> strokeWidthCalculator.calculate(textPositionAndWordImage.getWordImage().getImage()))
.filter(Double::isFinite)
.average()
.orElseThrow();
}
private static void insertResultIntoMap(int pageNumber, Map<Integer, List<OcrResultToWrite>> ocrResultToWritePerPage, FontStyleDetectionModel fontStyleDetectionModel) {
OcrResultToWrite ocrResult = OcrResultToWrite.fromFontStyleDetectionModel(fontStyleDetectionModel);
ocrResultToWritePerPage.compute(pageNumber, (key, existingList) -> {
if (existingList == null) {
return List.of(ocrResult);
} else {
return Stream.concat(existingList.stream(), Stream.of(ocrResult)).toList();
}
});
}
private void decideOnFontStyle(TextPositionAndWordImage textPositionsAndWordImage,
double standardStrokeWidth,
double standardTextHeight,
FontMetricsFactory boldFontMetricsFactory) {
double scaledStrokeWidth = scaleStrokeWidthByFontSize(textPositionsAndWordImage, standardStrokeWidth, standardTextHeight);
if (textPositionsAndWordImage.getWordImage().hasLargerStrokeWidth(scaledStrokeWidth)) {
textPositionsAndWordImage.getTextPositionInImage().setFontMetricsFactory(boldFontMetricsFactory);
textPositionsAndWordImage.getTextPositionInImage().setFontStyle(FontStyle.BOLD);
} else {
textPositionsAndWordImage.getTextPositionInImage().setFontStyle(FontStyle.REGULAR);
}
}
private static double scaleStrokeWidthByFontSize(TextPositionAndWordImage textPositionsAndWordImage, double standardStrokeWidth, double standardFontSize) {
double influenceOfFontSize = 1.0; // the paper states that stroke width scales exactly linearly with font size. This did not seem to be true for me. Maybe some of the preprocessing steps are affecting this.
double fontsizeScalingFactor = Math.sqrt(textPositionsAndWordImage.getWordImage().getTextHeight() / standardFontSize);
return standardStrokeWidth + (influenceOfFontSize * (fontsizeScalingFactor - 1) * standardStrokeWidth);
}
}
@@ -0,0 +1,57 @@
package com.knecon.fforesight.service.ocr.processor.service.scriptdetection;
import com.knecon.fforesight.service.ocr.processor.utils.ImageProcessingUtils;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.Sel;
import net.sourceforge.lept4j.util.LeptUtils;
/**
* This code is a good start for detecting italic text, although it has a few issues especially with glyphs which are naturally slanted. E.g. z, 2, 7, /
* If we want this maybe we should exclude these glyphs and then it might have less false positives. But in its current state i don't recommend using it.
*/
@NoArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ItalicDetector {
static String italicKernel = "ooxxooxxooxxoxxooXxooxxoxxooxxooxxoo";
Sel italicSel = Leptonica1.selCreateFromString(italicKernel, 9, 4, "italicKernel");
Sel brickSel = Leptonica1.selCreateBrick(3, 4, 1, 2, 1);
public boolean isItalic(Pix pix) {
Pix preprocessed = preprocess(pix);
Pix flipped = Leptonica1.pixFlipLR(null, pix);
Pix flippedPreprocessed = preprocess(flipped);
Leptonica1.pixFlipLR(flippedPreprocessed, flippedPreprocessed);
double pixelDensity = ImageProcessingUtils.calculatePixelDensity(preprocessed);
double flippedPixelDensity = ImageProcessingUtils.calculatePixelDensity(flippedPreprocessed);
LeptUtils.disposePix(preprocessed);
LeptUtils.disposePix(flipped);
LeptUtils.disposePix(flippedPreprocessed);
return flippedPixelDensity / pixelDensity < 0.85;
}
private Pix preprocess(Pix pix) {
Pix eroded = Leptonica1.pixErode(null, pix, italicSel.getPointer());
Pix dilated = Leptonica1.pixDilate(null, eroded, brickSel.getPointer());
LeptUtils.disposePix(eroded);
return dilated;
}
public void dispose() {
LeptUtils.dispose(italicSel);
LeptUtils.dispose(brickSel);
}
}
@@ -0,0 +1,58 @@
package com.knecon.fforesight.service.ocr.processor.service.scriptdetection;
import static net.sourceforge.lept4j.ILeptonica.L_THIN_FG;
import java.nio.IntBuffer;
import org.springframework.stereotype.Service;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.Sela;
import net.sourceforge.lept4j.util.LeptUtils;
@Service
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class StrokeWidthCalculator {
Sela thinningSel;
/**
* Uses a series of sels to thin all connected lines to a single pixel. Then the pixel ratio is a good estimation of the stroke width in pixels.
* <a href="http://www.leptonica.org/papers/conn.pdf">Leptonica Documentation on thinning</a>
* Since the baseline is a strokewidth of exactly one, we need to add 1 to the result.
*
* @param input binarized pix with text on it
* @return estimated stroke width in pixels
*/
public double calculate(Pix input) {
init();
Pix thinned = Leptonica1.pixThinConnectedBySet(input, L_THIN_FG, thinningSel, 0);
IntBuffer thinnedPixelCount = IntBuffer.allocate(1);
Leptonica1.pixCountPixels(thinned, thinnedPixelCount, null);
IntBuffer pixelCount = IntBuffer.allocate(1);
Leptonica1.pixCountPixels(input, pixelCount, null);
LeptUtils.disposePix(thinned);
return (double) pixelCount.get() / thinnedPixelCount.get() + 1;
}
private void init() {
if (thinningSel == null) {
thinningSel = Leptonica1.selaMakeThinSets(1, 0);
}
}
}
@@ -0,0 +1,65 @@
package com.knecon.fforesight.service.ocr.processor.service.threads;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.BlockingQueue;
import com.knecon.fforesight.service.ocr.processor.model.OcrImage;
import com.knecon.fforesight.service.ocr.processor.model.RenderedPageImageFile;
import com.knecon.fforesight.service.ocr.processor.model.UnprocessedImage;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import net.sourceforge.tess4j.TessAPI1;
/*
This just moves the Elements from the GhostScriptOutputListener into the ImageProcessing queue asynchronously
*/
@Slf4j
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class BlockingQueueFiller extends Thread {
final BlockingQueue<RenderedPageImageFile> imageInputQueue;
final BlockingQueue<UnprocessedImage> imageOutputQueue;
@Setter
boolean allImagesQueued;
@SneakyThrows
@Override
public void run() {
// Interrupting signals that the image extraction has finished
try {
while (!allImagesQueued) {
final UnprocessedImage image = imageInputQueue.take();
try {
imageOutputQueue.put(image);
} catch (InterruptedException e) {
imageOutputQueue.put(image);
}
}
} catch (InterruptedException e) {
log.info("All images extracted, emptying processing queue and stopping");
}
// empty the queue
try {
while (true) {
final UnprocessedImage image = imageInputQueue.remove();
imageOutputQueue.put(image);
}
} catch (NoSuchElementException e) {
log.debug("No images left in queue, stopping.");
}
}
}
@@ -0,0 +1,122 @@
package com.knecon.fforesight.service.ocr.processor.service.threads;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.knecon.fforesight.service.ocr.processor.model.RenderedPageImageFile;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class GhostScriptOutputHandler extends Thread {
static Pattern pageFinishedPattern = Pattern.compile("Page (\\d+)");
// If the stdError or stdOut buffer of a thread is not being emptied it might lock the process in case of errors, so we need to empty both streams to prevent a deadlock.
// Since both need to read simultaneously we need to implement the readers as separate threads.
final InputStream is;
final String processName;
final Type type;
final Map<Integer, RenderedPageImageFile> pagesToProcess;
final BlockingQueue<RenderedPageImageFile> renderedPageImageFileOutput;
int currentPageNumber;
public static GhostScriptOutputHandler errorHandler(InputStream is) {
return new GhostScriptOutputHandler(is, "GS", Type.ERROR, null, null);
}
public static GhostScriptOutputHandler stdOut(InputStream is,
Map<Integer, RenderedPageImageFile> pagesToProcess,
BlockingQueue<RenderedPageImageFile> renderedPageImageFileOutput) {
return new GhostScriptOutputHandler(is, "GS", Type.STD_OUT, pagesToProcess, renderedPageImageFileOutput);
}
@SneakyThrows
public void run() {
try (InputStreamReader isr = new InputStreamReader(is); BufferedReader br = new BufferedReader(isr)) {
String line;
while (true) {
line = br.readLine();
if (line == null) {
break;
}
if (type.equals(Type.ERROR)) {
log.error(processName + "_" + type.name() + ">" + line);
} else {
log.debug(processName + "_" + type.name() + ">" + line);
addProcessedImageToQueue(line);
}
}
}
is.close();
if (type.equals(Type.STD_OUT)) {
queueFinishedPage(currentPageNumber);
}
}
private void addProcessedImageToQueue(String line) {
/*
Ghostscript prints the pageNumber it is currently working on, so we remember the current page and queue it as soon as the next comes in.
*/
Matcher pageNumberMatcher = pageFinishedPattern.matcher(line);
if (pageNumberMatcher.find()) {
int pageNumber = Integer.parseInt(pageNumberMatcher.group(1));
if (currentPageNumber == 0) {
currentPageNumber = pageNumber;
return;
}
queueFinishedPage(currentPageNumber);
currentPageNumber = pageNumber;
}
}
private void queueFinishedPage(int pageNumber) {
var imageFile = this.pagesToProcess.get(pageNumber);
if (imageFile == null) {
throw new IllegalArgumentException(String.format("Page number %d does not exist in this thread. It only has pagenumbers %s", pageNumber, pagesToProcess.keySet()));
}
assert new File(imageFile.absoluteFilePath()).isFile();
renderedPageImageFileOutput.add(imageFile);
}
public enum Type {
ERROR,
STD_OUT
}
}
@@ -0,0 +1,110 @@
package com.knecon.fforesight.service.ocr.processor.service.threads;
import java.io.File;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import com.knecon.fforesight.service.ocr.processor.model.ExtractedImage;
import com.knecon.fforesight.service.ocr.processor.model.UnprocessedImage;
import com.knecon.fforesight.service.ocr.processor.service.ImageStreamEngine;
import com.knecon.fforesight.service.ocr.processor.service.OcrProgressLogger;
import com.knecon.fforesight.service.ocr.processor.service.Statistics;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ImageExtractionThread extends Thread {
static double FULL_PAGE_IMAGE_THRESHOLD = 0.99;
static double IMAGE_ALIGNMENT_THRESHOLD = 1;
int id;
@Getter
List<Integer> pageIndices;
File documentFile;
OcrProgressLogger logger;
Statistics stats;
OcrServiceSettings settings;
// output is written to these lists
BlockingQueue<UnprocessedImage> imageProcessingQueue;
List<Integer> stitchedPageNumbers;
@SneakyThrows
@Override
public void run() {
long timestamp;
for (Integer pageIndex : pageIndices) {
try (PDDocument document = Loader.loadPDF(documentFile)) { // load new PDDocument for thread safety, also keeps RAM usage low.
timestamp = System.currentTimeMillis();
List<ExtractedImage> extractedImages = getExtractedImages(pageIndex, document);
stats.increaseImageExtraction(id, System.currentTimeMillis() - timestamp);
if (extractedImages.isEmpty()) {
logger.logPageSkipped(pageIndex);
}
if (checkForFullPageOrStitchedImages(extractedImages, document.getPage(pageIndex - 1))) {
stitchedPageNumbers.add(pageIndex);
logger.addImagesToProcess(pageIndex, 0);
continue;
}
for (ExtractedImage image : extractedImages) {
imageProcessingQueue.put(image);
logger.addImagesToProcess(image.pageNumber(), image.numberOnPage());
}
}
}
}
private List<ExtractedImage> getExtractedImages(Integer pageIndex, PDDocument document) {
PDPage page = document.getPage(pageIndex - 1);
ImageStreamEngine imageStreamEngine = new ImageStreamEngine(settings);
imageStreamEngine.processPage(pageIndex, page);
return imageStreamEngine.getImagesOnCurrentPage();
}
@SneakyThrows
private boolean checkForFullPageOrStitchedImages(List<ExtractedImage> imagesOnCurrentPage, PDPage page) {
if (imagesOnCurrentPage.isEmpty()) {
return false;
}
for (ExtractedImage imageOnPage : imagesOnCurrentPage) {
if (imageOnPage.width() > FULL_PAGE_IMAGE_THRESHOLD * page.getCropBox().getWidth() && imageOnPage.height() > FULL_PAGE_IMAGE_THRESHOLD * page.getCropBox()
.getHeight()) {
return true;
}
}
//checking for intersections or direct alignment of images
for (int j = 0; j < imagesOnCurrentPage.size(); j++) {
for (int i = j + 1; i < imagesOnCurrentPage.size(); i++) {
if (imagesOnCurrentPage.get(j)
.getImageCoordinatesInInitialUserSpace()
.aligns(imagesOnCurrentPage.get(i).getImageCoordinatesInInitialUserSpace(), IMAGE_ALIGNMENT_THRESHOLD)) {
// TODO: see if we can stitch aligning images using BufferedImage and skip the gs conversion entirely
return true;
}
}
}
return false;
}
}
@@ -0,0 +1,250 @@
package com.knecon.fforesight.service.ocr.processor.service.threads;
import static net.sourceforge.tess4j.ITessAPI.TRUE;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.util.NoSuchElementException;
import java.util.concurrent.BlockingQueue;
import org.apache.pdfbox.pdmodel.PDDocument;
import com.knecon.fforesight.service.ocr.processor.model.ExtractedImage;
import com.knecon.fforesight.service.ocr.processor.model.ExtractedOcrImage;
import com.knecon.fforesight.service.ocr.processor.model.OcrImage;
import com.knecon.fforesight.service.ocr.processor.model.PageInformation;
import com.knecon.fforesight.service.ocr.processor.model.RenderedPageImageFile;
import com.knecon.fforesight.service.ocr.processor.model.RenderedPageOcrImage;
import com.knecon.fforesight.service.ocr.processor.model.UnprocessedImage;
import com.knecon.fforesight.service.ocr.processor.service.Statistics;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.utils.ImageProcessingUtils;
import com.sun.jna.ptr.PointerByReference;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import net.sourceforge.lept4j.L_Kernel;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.util.LeptUtils;
import net.sourceforge.tess4j.ITessAPI;
import net.sourceforge.tess4j.TessAPI1;
/*
* This thread does all the image processing. There should only be one, since Leptonica is not thread safe.
*/
@Slf4j
@RequiredArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class ImageProcessingThread extends Thread {
final BlockingQueue<UnprocessedImage> imageInputQueue;
final BlockingQueue<OcrImage> imageOutputQueue;
final ITessAPI.TessBaseAPI detectionScriptHandle = initDetectionScriptHandle();
final L_Kernel gaussianKernel = Leptonica1.makeGaussianKernel(2, 2, 1.2f, 1);
final Statistics stats;
final OcrServiceSettings settings;
final PDDocument document;
@Setter
boolean allImagesExtracted;
@SneakyThrows
@Override
public void run() {
try {
while (!allImagesExtracted) {
final UnprocessedImage image = imageInputQueue.take();
var ocrImage = this.process(image);
try {
imageOutputQueue.put(ocrImage);
} catch (InterruptedException e) {
imageOutputQueue.put(ocrImage);
}
}
} catch (InterruptedException e) {
log.info("All images extracted, emptying processing queue and stopping");
}
try {
while (true) {
final UnprocessedImage image = imageInputQueue.remove();
OcrImage ocrImage = this.process(image);
imageOutputQueue.put(ocrImage);
}
} catch (NoSuchElementException e) {
log.debug("No images left in processing queue, stopping.");
}
TessAPI1.TessBaseAPIEnd(this.detectionScriptHandle);
TessAPI1.TessBaseAPIDelete(this.detectionScriptHandle);
LeptUtils.dispose(gaussianKernel);
}
private OcrImage process(UnprocessedImage unprocessedImage) {
long timestamp = System.currentTimeMillis();
OcrImage ocrImage;
if (unprocessedImage instanceof ExtractedImage extractedImage) {
ocrImage = processExtractedImage(extractedImage);
} else if (unprocessedImage instanceof RenderedPageImageFile renderedPageImageFile) {
ocrImage = processRenderedPageImageFile(renderedPageImageFile);
} else {
throw new UnsupportedOperationException(String.format("Class %s is not supported!", unprocessedImage.getClass()));
}
stats.increaseImageProcessing(System.currentTimeMillis() - timestamp);
return ocrImage;
}
private OcrImage processRenderedPageImageFile(RenderedPageImageFile renderedPageImageFile) {
Pix pix = processPix(renderedPageImageFile.asPix(), settings.getDpi(), settings.getDpi());
int orientDegree = detectOrientation(pix, settings.getDpi(), detectionScriptHandle);
Pix rotatedPix = ImageProcessingUtils.deRotatePix(orientDegree, pix);
OcrImage ocrImage = new RenderedPageOcrImage(pix.h,
pix.w,
PageInformation.fromPDPage(renderedPageImageFile.pageNumber(), document.getPage(renderedPageImageFile.pageNumber() - 1)),
rotatedPix,
orientDegree);
if (pix != rotatedPix) {
LeptUtils.disposePix(pix);
}
return ocrImage;
}
private OcrImage processExtractedImage(ExtractedImage extractedImage) {
float imageDPI = Math.abs(extractedImage.image().getWidth() / (extractedImage.ctm().getScalingFactorX() / 72));
Pix pix = processPix(extractedImage.asPix(), imageDPI, settings.getDpi());
int orientDegree = detectOrientation(pix, settings.getDpi(), detectionScriptHandle);
Pix rotatedPix = ImageProcessingUtils.deRotatePix(orientDegree, pix);
OcrImage ocrImage = new ExtractedOcrImage(extractedImage.pageNumber(),
extractedImage.numberOnPage(),
extractedImage.height(),
extractedImage.width(),
extractedImage.ctm(),
rotatedPix,
pix.h,
pix.w,
orientDegree);
if (pix != rotatedPix) {
LeptUtils.disposePix(pix);
}
return ocrImage;
}
public int detectOrientation(Pix pix, int dpi, ITessAPI.TessBaseAPI detectionScriptHandle) {
TessAPI1.TessBaseAPISetImage2(detectionScriptHandle, pix);
TessAPI1.TessBaseAPISetSourceResolution(detectionScriptHandle, dpi);
IntBuffer orientationDegreeResultBuffer;
FloatBuffer orientationDegreeConfidenceBuffer;
PointerByReference scriptureNameBuffer;
FloatBuffer scriptureConfidenceBuffer;
orientationDegreeResultBuffer = IntBuffer.allocate(1);
orientationDegreeConfidenceBuffer = FloatBuffer.allocate(1);
scriptureNameBuffer = new PointerByReference(); // Is this memory being freed?
scriptureConfidenceBuffer = FloatBuffer.allocate(1);
int orientationDegree = 0;
int result = TessAPI1.TessBaseAPIDetectOrientationScript(detectionScriptHandle,
orientationDegreeResultBuffer,
orientationDegreeConfidenceBuffer,
scriptureNameBuffer,
scriptureConfidenceBuffer);
if (result == TRUE && orientationDegreeConfidenceBuffer.get() > settings.getMinRotationConfidence()) {
orientationDegree = orientationDegreeResultBuffer.get();
}
TessAPI1.TessBaseAPIClear(detectionScriptHandle);
return orientationDegree;
}
@SneakyThrows
private Pix processPix(Pix pix, float imageDpi, int targetDpi) {
Pix grayScale;
Pix scaledUp;
Pix gaussian;
Pix binarized;
//convert to grayscale
if (pix.d == 8) {
grayScale = pix;
} else if (pix.d == 32) {
grayScale = Leptonica1.pixConvertRGBToGrayFast(pix);
} else if (pix.d == 1) {
grayScale = Leptonica1.pixConvert1To8(null, pix, (byte) 0, (byte) 255);
} else {
throw new UnsupportedOperationException(String.format("Unknown pix format with bpp of %d", pix.d));
}
// scale up
float targetFactor = targetDpi / imageDpi;
if (targetFactor > 2.1) {
scaledUp = Leptonica1.pixScaleGray4xLI(grayScale);
} else if (targetFactor > 1.1) {
scaledUp = Leptonica1.pixScaleGray2xLI(grayScale);
} else {
scaledUp = grayScale;
}
// remove noise and prep for Otsu
gaussian = Leptonica1.pixConvolve(scaledUp, gaussianKernel, 8, 1);
// Threshold to binary
if (pix.w < 100 || pix.h < 100) {
binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
} else {
binarized = Leptonica1.pixOtsuThreshOnBackgroundNorm(gaussian, null, 50, 50, 165, 10, 100, 5, 5, 0.2f, null);
if (binarized == null) { // Sometimes Otsu just fails, then we binarize directly
binarized = Leptonica1.pixThresholdToBinary(gaussian, 170);
}
}
LeptUtils.disposePix(pix);
LeptUtils.disposePix(grayScale);
LeptUtils.disposePix(scaledUp);
LeptUtils.disposePix(gaussian);
return binarized;
}
private static ITessAPI.TessBaseAPI initDetectionScriptHandle() {
ITessAPI.TessBaseAPI handle = TessAPI1.TessBaseAPICreate();
String datapath = System.getenv("TESSDATA_PREFIX");
TessAPI1.TessBaseAPIInit3(handle, datapath, "osd");
TessAPI1.TessBaseAPISetVariable(handle, "debug_file", "/dev/null");
return handle;
}
}
@@ -0,0 +1,136 @@
package com.knecon.fforesight.service.ocr.processor.service.threads;
import static net.sourceforge.tess4j.ITessAPI.TRUE;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPICreate;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIInit1;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPISetPageSegMode;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPISetVariable;
import java.io.File;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.file.Path;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.BlockingQueue;
import com.knecon.fforesight.service.ocr.processor.settings.OcrServiceSettings;
import com.knecon.fforesight.service.ocr.processor.model.OcrImage;
import com.knecon.fforesight.service.ocr.processor.model.OcrResult;
import com.knecon.fforesight.service.ocr.processor.service.OcrProgressLogger;
import com.knecon.fforesight.service.ocr.processor.service.Statistics;
import com.knecon.fforesight.service.ocr.processor.utils.Tesseract2;
import com.sun.jna.StringArray;
import com.sun.jna.ptr.PointerByReference;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.util.LeptUtils;
import net.sourceforge.tess4j.ITessAPI;
import net.sourceforge.tess4j.ITesseract;
import net.sourceforge.tess4j.TessAPI1;
@Slf4j
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class OCRThread extends Thread {
int id;
BlockingQueue<OcrImage> imageInputQueue;
Path tesseractOutputDir;
List<OcrResult> results;
OcrProgressLogger logger;
Statistics stats;
OcrServiceSettings settings;
Tesseract2 instance;
public OCRThread(int id,
BlockingQueue<OcrImage> imageInputQueue,
Path tesseractOutputDir,
List<OcrResult> results,
OcrProgressLogger logger,
Statistics stats,
OcrServiceSettings settings) {
this.id = id;
this.imageInputQueue = imageInputQueue;
this.tesseractOutputDir = tesseractOutputDir;
this.results = results;
this.logger = logger;
this.stats = stats;
this.settings = settings;
this.instance = createInstance(settings);
}
@SneakyThrows
@Override
public void run() {
// Interrupting signals that the image extraction has finished
while (!isInterrupted()) {
try {
final OcrImage image = imageInputQueue.take();
this.process(image);
} catch (InterruptedException e) {
// set isInterrupted to true (This exception may only happen during active waiting for queue, and then isInterrupted will not be set!)
interrupt();
}
}
// empty the queue
try {
while (true) {
final OcrImage image = imageInputQueue.remove();
this.process(image);
}
} catch (NoSuchElementException e) {
log.debug("Executed tesseract on all Images, finishing.");
}
}
private void process(OcrImage image) {
long timestamp = System.currentTimeMillis();
String tmpOutputFileName = String.format("output_%04d_%04d", image.getPageNumber(), image.getNumberOnPage());
String tesseractOutputFileName = tesseractOutputDir.resolve(tmpOutputFileName).toFile().toString();
int psm = settings.getPsmOverride() < 0 ? image.getOptimalPageSegmentationMode() : settings.getPsmOverride();
executeTesseract(psm, image.getDpi(), image.getPix(), tesseractOutputFileName);
image.destroyPix();
results.add(OcrResult.create(image, tesseractOutputFileName));
logger.logImageFinished(image, psm);
stats.increaseTesseractDuration(id, System.currentTimeMillis() - timestamp);
}
@SneakyThrows
public void executeTesseract(int psm, int dpi, Pix pix, String tesseractOutputFileName) {
Leptonica1.pixWrite(tesseractOutputFileName + ".tiff", pix, 5); // write the used image for later bold detection
instance.setVariable("user_defined_dpi", String.valueOf(dpi));
instance.setPageSegMode(psm);
instance.createDocumentsWithResults(pix, null, tesseractOutputFileName, List.of(ITesseract.RenderedFormat.HOCR), ITessAPI.TessPageIteratorLevel.RIL_BLOCK);
}
private static Tesseract2 createInstance(OcrServiceSettings settings) {
Tesseract2 instance = new Tesseract2();
instance.setVariable("debug_file", "/dev/null"); // remove warnings from std out
instance.setOcrEngineMode(1); // set to LSTM based Engine
instance.setLanguage(settings.getLanguages());
return instance;
}
}
@@ -0,0 +1,30 @@
package com.knecon.fforesight.service.ocr.processor.settings;
import org.apache.pdfbox.cos.COSName;
import org.springframework.boot.context.properties.ConfigurationProperties;
import lombok.AccessLevel;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@ConfigurationProperties("ocr-service")
@FieldDefaults(level = AccessLevel.PRIVATE)
public class OcrServiceSettings {
int ocrThreadCount = 4; // Number of OCR threads
int imageExtractThreadCount = 2; // Number of image extraction threads
int gsProcessCount = 1; // Number of Ghostscript processes
int dpi = 300; // Target DPI for binarized images
int psmOverride = -1; // Overrides the page segmentation mode if > 0
int minImageHeight = 20; // Minimum height for images to be processed
int minImageWidth = 20; // Minimum width for images to be processed
float minRotationConfidence = 2; // Sets a lower bound for the confidence rating for rotated pages.
boolean debug; // If true, overlays OCR images with a grid and draws word bounding boxes
boolean removeWatermark; // If true, watermarks will be removed
String languages = "deu+eng"; // Defines languages loaded into Tesseract as 3-char codes, additional languages must also be installed in the docker environment
COSName ocrMarkedContentTag = COSName.getPDFName("KNECON_OCR");
boolean boldDetection = true; // if true, bold detection will be attempted
double boldThreshold = 0.5; // Words are opened with a brick of average stroke width, if the ratio of remaining pixels is higher the word is determined bold.
}
@@ -0,0 +1,85 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import java.awt.AlphaComposite;
import java.awt.Color;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.Transparency;
import java.awt.image.BufferedImage;
import java.nio.IntBuffer;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceGray;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import com.knecon.fforesight.service.ocr.processor.model.ExtractedImage;
import com.sun.jna.ptr.PointerByReference;
import lombok.SneakyThrows;
import lombok.experimental.UtilityClass;
import net.sourceforge.lept4j.L_Kernel;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.lept4j.util.LeptUtils;
@UtilityClass
public class ImageProcessingUtils {
public BufferedImage convertToDeviceColorSpace(ExtractedImage extractedImage) {
BufferedImage image;
if (extractedImage.colorSpace() instanceof PDDeviceRGB || extractedImage.colorSpace() instanceof PDDeviceGray) {
image = extractedImage.image();
} else {
BufferedImage pdfImage = extractedImage.image();
image = new BufferedImage(pdfImage.getWidth(), pdfImage.getHeight(), BufferedImage.TYPE_BYTE_GRAY);
Graphics g = image.getGraphics();
g.drawImage(pdfImage, 0, 0, null);
g.dispose();
}
return image;
}
public Pix deRotatePix(int orientDegree, Pix pix) {
return switch (360 - orientDegree) {
case 90 -> Leptonica1.pixRotateOrth(pix, 1);
case 180 -> Leptonica1.pixRotateOrth(pix, 2);
case 270 -> Leptonica1.pixRotateOrth(pix, 3);
default -> pix;
};
}
public static void setAlphaChannelToWhite(BufferedImage image) {
if (image.getTransparency() == Transparency.TRANSLUCENT) {
// NOTE: For BITMASK images, the color model is likely IndexColorModel,
// and this model will contain the "real" color of the transparent parts
// which is likely a better fit than unconditionally setting it to white.
// Fill background with white
Graphics2D graphics = image.createGraphics();
try {
graphics.setComposite(AlphaComposite.DstOver); // Set composite rules to paint "behind"
graphics.setPaint(Color.WHITE);
graphics.fillRect(0, 0, image.getWidth(), image.getHeight());
} finally {
graphics.dispose();
}
}
}
public static double calculatePixelDensity(Pix pix) {
IntBuffer pixelCount = IntBuffer.allocate(1);
int result = Leptonica1.pixCountPixels(pix, pixelCount, null);
if (result == 0) {
return (double) pixelCount.get() / (pix.h * pix.w);
} else {
return -1;
}
}
}
@@ -0,0 +1,73 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import lombok.experimental.UtilityClass;
import net.sourceforge.lept4j.L_Kernel;
import net.sourceforge.lept4j.Leptonica1;
@UtilityClass
public class KernelUtils {
/*
-1, -1, -1
-1, 8, -1
-1, -1, -1
*/
public L_Kernel createFullLaplacianKernel() {
L_Kernel laplacianKernel = Leptonica1.kernelCreate(3, 3);
Leptonica1.kernelSetElement(laplacianKernel, 0, 0, -1);
Leptonica1.kernelSetElement(laplacianKernel, 0, 1, -1);
Leptonica1.kernelSetElement(laplacianKernel, 0, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 0, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 0, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 1, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 1, 8);
return laplacianKernel;
}
/*
0, 0, -1, 0, 0
0, -1, -1, -1, 0
-1, -1, 12, -1, -1
0, -1, -1, -1, 0
0, 0, -1, 0, 0
*/
public L_Kernel createLaplacianKernel5x5() {
L_Kernel laplacianKernel = Leptonica1.kernelCreate(5, 5);
Leptonica1.kernelSetElement(laplacianKernel, 0, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 1, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 3, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 0, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 1, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 3, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 4, -1);
Leptonica1.kernelSetElement(laplacianKernel, 3, 1, -1);
Leptonica1.kernelSetElement(laplacianKernel, 3, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 3, 3, -1);
Leptonica1.kernelSetElement(laplacianKernel, 4, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 2, 12);
return laplacianKernel;
}
/*
0, -1, 0
-1, 4, -1
0, -1, 0
*/
public L_Kernel createLaplacianKernel() {
L_Kernel laplacianKernel = Leptonica1.kernelCreate(3, 3);
Leptonica1.kernelSetElement(laplacianKernel, 0, 1, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 0, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 2, -1);
Leptonica1.kernelSetElement(laplacianKernel, 2, 1, -1);
Leptonica1.kernelSetElement(laplacianKernel, 1, 1, 4);
return laplacianKernel;
}
}
@@ -0,0 +1,64 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.IntStream;
import lombok.experimental.UtilityClass;
@UtilityClass
public class ListSplittingUtils {
public List<List<Integer>> buildBalancedContinuousSublist(Integer totalNumberOfEntries, int threadCount) {
return buildBalancedSublist(IntStream.range(0, totalNumberOfEntries).map(i -> i + 1).boxed().toList(), threadCount);
}
public <T> List<List<T>> buildBalancedSublist(List<T> entries, int threadCount) {
List<Integer> balancedEntryCounts = buildBalancedEntryCounts(entries.size(), threadCount);
List<List<T>> balancedSublist = new ArrayList<>(threadCount);
int startIdx = 0;
for (Integer numberOfEntriesPerThread : balancedEntryCounts) {
balancedSublist.add(entries.subList(startIdx, startIdx + numberOfEntriesPerThread));
startIdx += numberOfEntriesPerThread;
}
return balancedSublist;
}
public <T> List<List<List<T>>> buildBatchedBalancedSublist(List<T> entries, int threadCount, int batchSize) {
// batches -> threads -> entries
List<List<List<T>>> batchedBalancedSubList = new LinkedList<>();
List<List<List<T>>> threadsWithBatches = buildBalancedSublist(entries, threadCount).stream().map(list -> buildBalancedSublist(list, batchSize)).toList();
// swap first two dimensions
for (int batchIdx = 0; batchIdx < batchSize; batchIdx++) {
List<List<T>> threadEntriesPerBatch = new ArrayList<>(threadCount);
for (int threadIdx = 0; threadIdx < threadCount; threadIdx++) {
threadEntriesPerBatch.add(threadsWithBatches.get(threadIdx).get(batchIdx));
}
batchedBalancedSubList.add(threadEntriesPerBatch);
}
return batchedBalancedSubList;
}
public List<Integer> buildBalancedEntryCounts(int totalNumberOfEntries, int threadCount) {
List<Integer> numberOfPagesPerThread = new ArrayList<>(threadCount);
for (int i = 0; i < threadCount; i++) {
numberOfPagesPerThread.add(0);
}
int threadIdx;
for (int i = 0; i < totalNumberOfEntries; i++) {
threadIdx = i % threadCount;
numberOfPagesPerThread.set(threadIdx, numberOfPagesPerThread.get(threadIdx) + 1);
}
return numberOfPagesPerThread;
}
}
@@ -0,0 +1,21 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import java.awt.geom.AffineTransform;
import com.knecon.fforesight.service.ocr.processor.model.QuadPoint;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
import lombok.experimental.UtilityClass;
@UtilityClass
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class PdfDpiCalculator {
public int calculateDpi(QuadPoint imageBounds, AffineTransform imageCTM, double width) {
QuadPoint transformedImageBounds = imageBounds.getTransformed(imageCTM);
double transformedWidth = transformedImageBounds.a().distance(transformedImageBounds.d());
double widthInInches = transformedWidth * 1 / 72;
return (int) Math.ceil(width / widthInInches);
}
}
@@ -1,4 +1,4 @@
package com.iqser.red.service.ocr.v1.server.utils;
package com.knecon.fforesight.service.ocr.processor.utils;
import com.pdftron.pdf.ColorPt;
import com.pdftron.pdf.ColorSpace;
@@ -0,0 +1,168 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import static net.sourceforge.tess4j.ITessAPI.TRUE;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIDelete;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIEnd;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIGetIterator;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIGetStringVariable;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIMeanTextConf;
import static net.sourceforge.tess4j.TessAPI1.TessBaseAPIProcessPage;
import static net.sourceforge.tess4j.TessAPI1.TessDeleteResultRenderer;
import static net.sourceforge.tess4j.TessAPI1.TessHOcrRendererCreate;
import static net.sourceforge.tess4j.TessAPI1.TessPageIteratorBegin;
import static net.sourceforge.tess4j.TessAPI1.TessPageIteratorBoundingBox;
import static net.sourceforge.tess4j.TessAPI1.TessPageIteratorNext;
import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorConfidence;
import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorDelete;
import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorGetPageIterator;
import static net.sourceforge.tess4j.TessAPI1.TessResultIteratorGetUTF8Text;
import static net.sourceforge.tess4j.TessAPI1.TessResultRendererBeginDocument;
import static net.sourceforge.tess4j.TessAPI1.TessResultRendererEndDocument;
import static net.sourceforge.tess4j.TessAPI1.TessResultRendererInsert;
import java.awt.Rectangle;
import java.nio.IntBuffer;
import java.util.ArrayList;
import java.util.List;
import com.sun.jna.Pointer;
import lombok.extern.slf4j.Slf4j;
import net.sourceforge.lept4j.Pix;
import net.sourceforge.tess4j.ITessAPI;
import net.sourceforge.tess4j.OCRResult;
import net.sourceforge.tess4j.TessAPI1;
import net.sourceforge.tess4j.Tesseract;
import net.sourceforge.tess4j.TesseractException;
import net.sourceforge.tess4j.Word;
@Slf4j
/**
* Overriden version only so I can use Tesseract1 with Pixs instead of BufferedImages. All Functions are copied and then the BufferedImage -> Pix conversion deleted.
*/ public class Tesseract2 extends Tesseract {
private int createDocuments(Pix pix, String filename, ITessAPI.TessResultRenderer renderer) {
String title = TessBaseAPIGetStringVariable(getHandle(), DOCUMENT_TITLE);
TessResultRendererBeginDocument(renderer, title);
int result = TessBaseAPIProcessPage(getHandle(), pix, 0, filename, null, 0, renderer);
TessResultRendererEndDocument(renderer);
// if (result == ITessAPI.FALSE) {
// throw new TesseractException("Error during processing page.");
// }
return TessBaseAPIMeanTextConf(getHandle());
}
public OCRResult createDocumentsWithResults(Pix bi, String filename, String outputbase, List<RenderedFormat> formats, int pageIteratorLevel) throws TesseractException {
List<OCRResult> results = createDocumentsWithResults(new Pix[]{bi}, new String[]{filename}, new String[]{outputbase}, formats, pageIteratorLevel);
if (!results.isEmpty()) {
return results.get(0);
} else {
return null;
}
}
public List<OCRResult> createDocumentsWithResults(Pix[] pixs, String[] filenames, String[] outputbases, List<RenderedFormat> formats, int pageIteratorLevel) {
if (pixs.length != filenames.length || pixs.length != outputbases.length) {
throw new RuntimeException("The three arrays must match in length.");
}
init();
setVariables();
List<OCRResult> results = new ArrayList<OCRResult>();
try {
for (int i = 0; i < pixs.length; i++) {
try {
ITessAPI.TessResultRenderer renderer = createRenderers(outputbases[i], formats);
int meanTextConfidence = createDocuments(pixs[i], filenames[i], renderer);
TessDeleteResultRenderer(renderer);
List<Word> words = meanTextConfidence > 0 ? getRecognizedWords(pageIteratorLevel) : new ArrayList<Word>();
results.add(new OCRResult(meanTextConfidence, words));
} catch (Exception e) {
// skip the problematic image file
log.warn(e.getMessage(), e);
}
}
} finally {
dispose();
}
return results;
}
private List<Word> getRecognizedWords(int pageIteratorLevel) {
List<Word> words = new ArrayList<>();
try {
ITessAPI.TessResultIterator ri = TessBaseAPIGetIterator(getHandle());
ITessAPI.TessPageIterator pi = TessResultIteratorGetPageIterator(ri);
TessPageIteratorBegin(pi);
do {
Pointer ptr = TessResultIteratorGetUTF8Text(ri, pageIteratorLevel);
if (ptr == null) {
continue;
}
String text = ptr.getString(0);
TessAPI1.TessDeleteText(ptr);
float confidence = TessResultIteratorConfidence(ri, pageIteratorLevel);
IntBuffer leftB = IntBuffer.allocate(1);
IntBuffer topB = IntBuffer.allocate(1);
IntBuffer rightB = IntBuffer.allocate(1);
IntBuffer bottomB = IntBuffer.allocate(1);
TessPageIteratorBoundingBox(pi, pageIteratorLevel, leftB, topB, rightB, bottomB);
int left = leftB.get();
int top = topB.get();
int right = rightB.get();
int bottom = bottomB.get();
Word word = new Word(text, confidence, new Rectangle(left, top, right - left, bottom - top));
words.add(word);
} while (TessPageIteratorNext(pi, pageIteratorLevel) == TRUE);
// TessPageIteratorDelete(pi);
TessResultIteratorDelete(ri);
} catch (Exception e) {
log.warn(e.getMessage(), e);
}
return words;
}
private ITessAPI.TessResultRenderer createRenderers(String outputbase, List<RenderedFormat> formats) {
ITessAPI.TessResultRenderer renderer = null;
for (RenderedFormat format : formats) {
switch (format) {
case HOCR:
if (renderer == null) {
renderer = TessHOcrRendererCreate(outputbase);
} else {
TessResultRendererInsert(renderer, TessHOcrRendererCreate(outputbase));
}
break;
}
}
return renderer;
}
@Override
protected void dispose() {
TessBaseAPIEnd(getHandle());
TessBaseAPIDelete(getHandle());
}
}
@@ -0,0 +1,29 @@
package com.knecon.fforesight.service.ocr.processor.service;
import java.io.File;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.Test;
import com.knecon.fforesight.service.ocr.processor.model.FontMetrics;
import com.knecon.fforesight.service.ocr.processor.service.fonts.Type0FontMetricsFactory;
import lombok.SneakyThrows;
import lombok.experimental.SuperBuilder;
@SuppressWarnings("PMD")
class Type0FontMetricsFactoryTest {
@Test
@SneakyThrows
public void testStringWidth() {
try (PDDocument document = Loader.loadPDF(new File(Type0FontMetricsFactoryTest.class.getClassLoader().getResource("InvisibleText.pdf").getPath()))) {
Type0FontMetricsFactory metricsFactory = Type0FontMetricsFactory.regular(document);
FontMetrics fontMetrics = metricsFactory.calculateMetrics("deine mutter", 100, 50);
}
}
}
@@ -0,0 +1,36 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import static net.sourceforge.lept4j.ILeptonica.IFF_PNG;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import net.sourceforge.lept4j.Leptonica1;
import net.sourceforge.lept4j.Pix;
@Disabled
class ImageProcessingUtilsTest {
@BeforeEach
public void loadLeptonica() {
System.setProperty("jna.library.path", System.getenv("VCPKG_DYNAMIC_LIB"));
}
@Test
public void testRotation() {
Pix pix = Leptonica1.pixRead("/home/kschuettler/Downloads/painHarold.webp");
Pix pix2 = ImageProcessingUtils.deRotatePix(0, pix);
Leptonica1.pixWrite("/tmp/0.png", pix2, IFF_PNG);
Pix pix3 = ImageProcessingUtils.deRotatePix(90, pix);
Leptonica1.pixWrite("/tmp/90.png", pix3, IFF_PNG);
Pix pix4 = ImageProcessingUtils.deRotatePix(180, pix);
Leptonica1.pixWrite("/tmp/180.png", pix4, IFF_PNG);
Pix pix5 = ImageProcessingUtils.deRotatePix(270, pix);
Leptonica1.pixWrite("/tmp/270.png", pix5, IFF_PNG);
}
}
@@ -0,0 +1,21 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.Collection;
import org.junit.jupiter.api.Test;
class ListSplittingUtilsTest {
@Test
public void testBalancedListSplitting() {
int threadCount = 18;
int numberOfPages = 48;
var balancedList = ListSplittingUtils.buildBalancedContinuousSublist(numberOfPages, threadCount);
assertEquals(threadCount, balancedList.size());
assertEquals(numberOfPages, balancedList.stream().mapToLong(Collection::size).sum());
}
}
@@ -0,0 +1,83 @@
package com.knecon.fforesight.service.ocr.processor.utils;
import java.awt.image.BufferedImage;
import java.io.File;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.IntStream;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.PDFRenderer;
import org.apache.pdfbox.tools.imageio.ImageIOUtil;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import org.springframework.util.FileSystemUtils;
import com.knecon.fforesight.service.ocr.processor.service.OsUtils;
import com.knecon.fforesight.service.ocr.processor.service.threads.GhostScriptOutputHandler;
import lombok.SneakyThrows;
// YOU NEED GHOSTSCRIPT INSTALLED TO RUN THIS TEST!!!!
@Disabled
public class Pdf2ImgTest {
private static final int DPI = 150;
@Test
@SneakyThrows
@Disabled
public void testPDFBox() {
String outputDir = OsUtils.getTemporaryDirectory("imageOutput", "");
new File(outputDir).mkdirs();
ClassPathResource resource = new ClassPathResource("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
try (PDDocument document = Loader.loadPDF(resource.getFile())) {
PDFRenderer renderer = new PDFRenderer(document);
for (int pageNumber = 0; pageNumber < document.getNumberOfPages(); pageNumber++) {
BufferedImage image = renderer.renderImageWithDPI(pageNumber, DPI);
boolean written = ImageIOUtil.writeImage(image, "tif", new File(outputDir + String.format("page%04d", pageNumber)).getAbsolutePath(), DPI);
System.out.printf("%d: %s%n", pageNumber, written);
}
}
FileSystemUtils.deleteRecursively(new File(outputDir));
}
@Test
@SneakyThrows
public void testGhostScriptParallel() {
int numOfProcesses = 5;
ClassPathResource resource = new ClassPathResource("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
String outputDir = "/tmp/ghostscript_out/";
List<Process> processes = IntStream.range(0, numOfProcesses).boxed().parallel().map(i -> buildCmdArgs(i, outputDir, resource)).map(Pdf2ImgTest::executeProcess).toList();
List<Integer> processExitCodes = new LinkedList<>();
for (Process process : processes) {
processExitCodes.add(process.waitFor());
}
System.out.println("Ghostscripts finished with exit codes " + processExitCodes);
FileSystemUtils.deleteRecursively(new File(outputDir));
}
@SneakyThrows
private static Process executeProcess(String[] cmdArgs) {
return Runtime.getRuntime().exec(cmdArgs);
}
@SneakyThrows
private static String[] buildCmdArgs(Integer i, String outputDir, ClassPathResource resource) {
String outDir = outputDir + "/" + i + "/";
new File(outDir).mkdirs();
return new String[]{"gs", "-dNOPAUSE", "-sDEVICE=tiffgray", "-r" + DPI, "-sOutputFile=" + outDir + "page%04d", resource.getFile().toString(), "-c", "quit"};
}
}
@@ -1,160 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.iqser.red.service</groupId>
<artifactId>ocr-service-v1</artifactId>
<version>3.0-SNAPSHOT</version>
</parent>
<artifactId>ocr-service-server-v1</artifactId>
<properties>
<tennat-commons.version>0.10.0</tennat-commons.version>
<persistence-service.version>2.93.0</persistence-service.version>
</properties>
<dependencies>
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>tenant-commons</artifactId>
<version>${tennat-commons.version}</version>
</dependency>
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-internal-api-v1</artifactId>
<version>${persistence-service.version}</version>
</dependency>
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>ocr-service-api-v1</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>storage-commons</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>pdftron-logic-commons</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>spring-commons</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>metric-commons</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
<dependency>
<groupId>com.pdftron</groupId>
<artifactId>PDFNet</artifactId>
<version>10.1.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-kms</artifactId>
<version>1.12.440</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<!-- Test -->
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>test-commons</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.amqp</groupId>
<artifactId>spring-rabbit-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<!-- generate git.properties for exposure in /info -->
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
<configuration>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<gitDescribe>
<tags>true</tags>
</gitDescribe>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<annotationProcessors>
<annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
</annotationProcessors>
</configuration>
</plugin>
<plugin>
<!-- repackages the generated jar into a runnable fat-jar and makes it
executable -->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<executable>true</executable>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>pdftron</id>
<name>PDFNet Maven</name>
<url>https://pdftron.com/maven/release</url>
</repository>
</repositories>
</project>
@@ -1,16 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class ImagePosition {
private Rectangle rectangle;
private boolean hasTransparency;
}
@@ -1,14 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
@Data
public class Classification {
private Map<String, Float> probabilities = new HashMap<>();
private String label;
}
@@ -1,11 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class FilterGeometry {
private ImageSize imageSize;
private ImageFormat imageFormat;
}
@@ -1,12 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class Filters {
private FilterGeometry geometry;
private Probability probability;
private boolean allPassed;
}
@@ -1,11 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class Geometry {
private float width;
private float height;
}
@@ -1,12 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class ImageFormat {
private float quotient;
private boolean tooTall;
private boolean tooWide;
}
@@ -1,14 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class ImageMetadata {
private Classification classification;
private Position position;
private Geometry geometry;
private Filters filters;
private boolean alpha;
}
@@ -1,26 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@Data
public class ImageServiceResponse {
private String dossierId;
private String fileId;
@JsonProperty(value = "imageMetadata")
@JsonAlias("data")
private List<ImageMetadata> data = new ArrayList<>();
@JsonProperty(value = "imageMetadata")
@JsonAlias("data")
public void setData(List<ImageMetadata> data) {this.data = data;}
}
@@ -1,12 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class ImageSize {
private float quotient;
private boolean tooLarge;
private boolean tooSmall;
}
@@ -1,14 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class Position {
private float x1;
private float x2;
private float y1;
private float y2;
private int pageNumber;
}
@@ -1,10 +0,0 @@
package com.iqser.red.service.ocr.v1.server.model.image;
import lombok.Data;
@Data
public class Probability {
private boolean unconfident;
}
@@ -1,196 +0,0 @@
package com.iqser.red.service.ocr.v1.server.service;
import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D;
import java.awt.geom.RectangularShape;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.springframework.stereotype.Service;
import com.pdftron.common.Matrix2D;
import com.pdftron.common.PDFNetException;
import com.pdftron.pdf.Element;
import com.pdftron.pdf.ElementReader;
import com.pdftron.pdf.PDFDoc;
import com.pdftron.pdf.Page;
import com.pdftron.pdf.Rect;
import com.pdftron.pdf.RectCollection;
import lombok.SneakyThrows;
@Service
public class ImagePositionRetrievalService {
private static final double TOLERANCE = 1e-1;
// any image with smaller height and width than this gets thrown out, see everyPointInDashedLineIsImage.pdf
private static final int PIXEL_THRESHOLD = 10;
/**
* Iterates over all elements in a PDF Document and retrieves the bounding box for each image, that is larger than the pixel threshold of 10 in either dimension.
* Then it adjusts the bounding boxes for the page rotation.
* If the mirrorY flag is set, the Y Coordinates are mirrored and moved up by the page height. This is required for PDFTrons OCRModule.
*
* @param pdfDoc a PDF File as PDFTron PDFDoc class
* @param mirrorY if this flag is set, all coordinates are calculated with upper left corner as (0,0), else initial user space
* @return a map with the page indices as keys and the image bounding boxes on that page as a RectCollection
*/
@SneakyThrows
public Map<Integer, RectCollection> getImagePositionPerPage(PDFDoc pdfDoc, boolean mirrorY) {
Map<Integer, RectCollection> pageIdToImagePositions = new HashMap<>();
ElementReader reader = new ElementReader();
for (int pageId = 1; pageId <= pdfDoc.getPageCount(); ++pageId) {
RectCollection imagePositions = new RectCollection();
reader.begin(pdfDoc.getPage(pageId));
findImagePositionsOnPage(reader, imagePositions, pdfDoc.getPage(pageId), mirrorY);
imagePositions = mergeOverlappingRects(imagePositions);
reader.end();
if (imagePositions.getNumRects() > 0) {
pageIdToImagePositions.put(pageId, imagePositions);
}
}
reader.destroy();
return pageIdToImagePositions;
}
private void findImagePositionsOnPage(ElementReader reader, RectCollection imagePositions, Page currentPage, boolean mirrorY) throws PDFNetException {
Element element;
while ((element = reader.next()) != null) {
switch (element.getType()) {
case Element.e_image, Element.e_inline_image -> {
// see everyPointInDashedLineIsImage.pdf TestFile
if (element.getImageHeight() > PIXEL_THRESHOLD || element.getImageWidth() > PIXEL_THRESHOLD) {
imagePositions.addRect(toRotationAdjustedRect(element.getBBox(), currentPage, mirrorY));
}
}
case Element.e_form -> {
reader.formBegin();
findImagePositionsOnPage(reader, imagePositions, currentPage, mirrorY);
reader.end();
}
}
}
}
@SneakyThrows
public RectCollection mergeOverlappingRects(RectCollection imagePositions) {
if (imagePositions.getNumRects() < 2) {
return imagePositions;
}
List<Rectangle2D> rectangleList = toSortedRectangleList(imagePositions);
mergeRectangleList(rectangleList);
return toRectCollection(rectangleList);
}
// Sometimes images are split up into stripes, here we merge the positions of aligned and intersecting rectangles into one larger rectangle
private void mergeRectangleList(List<Rectangle2D> rectangleList) {
for (int idx = 0; rectangleList.size() >= idx + 2; ) {
var rect1 = rectangleList.get(idx);
var rect2 = rectangleList.get(idx + 1);
if (intersects(rect1, rect2) && isAlignedXOrY(rect1, rect2)) {
rectangleList.remove(idx + 1);
rectangleList.remove(idx);
rectangleList.add(idx, rect1.createUnion(rect2));
} else {
++idx;
}
}
}
private boolean intersects(Rectangle2D rect1, Rectangle2D rect2) {
return rect1.intersects(rect2.getMinX() - TOLERANCE, rect2.getMinY() - TOLERANCE, rect2.getWidth() + (2 * TOLERANCE), rect2.getHeight() + (2 * TOLERANCE));
}
private boolean isAlignedXOrY(Rectangle2D rect1, Rectangle2D rect2) {
boolean isAlignedX = Math.abs(rect1.getMinX() - rect2.getMinX()) < TOLERANCE && Math.abs(rect1.getMaxX() - rect2.getMaxX()) < TOLERANCE;
boolean isAlignedY = Math.abs(rect1.getMinY() - rect2.getMinY()) < TOLERANCE && Math.abs(rect1.getMaxY() - rect2.getMaxY()) < TOLERANCE;
return isAlignedX || isAlignedY;
}
private Rect toRotationAdjustedRect(Rect bbox, Page page, boolean mirrorY) throws PDFNetException {
int rotation = page.getRotation();
double height = page.getPageHeight();
double width = page.getPageWidth();
// Even though PDFTron almost always has the origin in the lower left corner, for some reason, the OCRModule's addTextZonesForPage() uses the upper left corner as origin...
Matrix2D mirrorMatrix;
if (mirrorY) {
mirrorMatrix = new Matrix2D(1, 0, 0, -1, 0, height);
} else {
mirrorMatrix = new Matrix2D();
}
// We need to rotate the rects to fit to the page rotation
Matrix2D rotationMatrix = switch (rotation) {
case 1 -> new Matrix2D(0, -1, 1, 0, 0, height);
case 2 -> new Matrix2D(-1, 0, 0, -1, width, height);
case 3 -> new Matrix2D(0, 1, -1, 0, width, 0);
default -> new Matrix2D();
};
Matrix2D finalMatrix = mirrorMatrix.multiply(rotationMatrix);
Point2D.Double p1 = finalMatrix.multPoint(bbox.getX1(), bbox.getY1());
Point2D.Double p2 = finalMatrix.multPoint(bbox.getX2(), bbox.getY2());
// PDFTron Rect *needs* lower left and upper right coordinates to calculate width and height correctly, even though the documentation states otherwise
Point2D.Double lowerLeft = new Point2D.Double(Math.min(p1.x, p2.x), Math.min(p1.y, p2.y));
Point2D.Double upperRight = new Point2D.Double(Math.max(p1.x, p2.x), Math.max(p1.y, p2.y));
return new Rect(lowerLeft.x, lowerLeft.y, upperRight.x, upperRight.y);
}
private RectCollection toRectCollection(List<Rectangle2D> rectangleList) {
RectCollection rectCollection = new RectCollection();
rectangleList.forEach(r -> {
try {
rectCollection.addRect(new Rect(r.getMinX(), r.getMinY(), r.getMaxX(), r.getMaxY()));
} catch (PDFNetException e) {
throw new RuntimeException(e);
}
});
return rectCollection;
}
@SneakyThrows
private List<Rectangle2D> toSortedRectangleList(RectCollection rectCollection) {
List<Rectangle2D> list = new LinkedList<>();
for (int i = 0; i < rectCollection.getNumRects(); ++i) {
Rect r = rectCollection.getRectAt(i);
list.add(new Rectangle2D.Double(r.getX1(), r.getY1(), r.getWidth(), r.getHeight()));
}
list.sort(Comparator.comparingDouble(RectangularShape::getMinY).thenComparing(RectangularShape::getMinX));
return list;
}
}
@@ -1,196 +0,0 @@
package com.iqser.red.service.ocr.v1.server.service;
import static java.lang.String.format;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
import com.iqser.red.service.ocr.v1.api.model.OCRStatusUpdateResponse;
import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
import com.pdftron.common.PDFNetException;
import com.pdftron.pdf.OCRModule;
import com.pdftron.pdf.OCROptions;
import com.pdftron.pdf.Optimizer;
import com.pdftron.pdf.PDFDoc;
import com.pdftron.pdf.Page;
import com.pdftron.pdf.RectCollection;
import com.pdftron.sdf.SDFDoc;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class OCRService {
public static final String ENGLISH = "eng";
private final FileStorageService fileStorageService;
private final OcrServiceSettings settings;
private final RabbitTemplate rabbitTemplate;
private final WatermarkRemovalService watermarkRemovalService;
private final InvisibleElementRemovalService invisibleElementRemovalService;
private final ImagePositionRetrievalService imagePositionRetrievalService;
/**
* First loads the PDF Document from storage.
* Then removes all invisible Elements from the PDF, check InvisibleElementRemovalService for details.
* Then gets Image Position Information, check ImagePositionRetrievalService for details.
* Then runs OCR page by page, exclusively on pages which have images on them. It does so, by creating a new PDFDoc and inserting a single page at a time.
* This is because PDFTron OCROptions overlays all regions where OCR should not be run with white images. It does not check for empty pages.
* For Documents with many pages but few Images this results in major performance improvements.
* It then re-adds the OCRed Pages to the original document and saves it.
*
* @param dossierId The dossier id
* @param fileId The file id
* @param out OutputStream to write the file to
*/
@Timed("redactmanager_runOcrOnDocument")
public void runOcrOnDocument(String dossierId, String fileId, OutputStream out) throws IOException {
ByteArrayOutputStream transferOutputStream = new ByteArrayOutputStream();
InputStream fileStream = fileStorageService.getOriginalFileAsStream(dossierId, fileId);
try {
if (settings.isRemoveWatermark()) {
watermarkRemovalService.removeWatermarks(fileStream, transferOutputStream);
fileStream.close();
fileStream = new ByteArrayInputStream(transferOutputStream.toByteArray());
transferOutputStream.close();
transferOutputStream = new ByteArrayOutputStream();
}
invisibleElementRemovalService.removeInvisibleElements(fileStream, transferOutputStream, false);
try (InputStream transferInputStream = new ByteArrayInputStream(transferOutputStream.toByteArray())) {
long ocrStart = System.currentTimeMillis();
runOcr(transferInputStream, out, fileId);
long ocrEnd = System.currentTimeMillis();
log.info("ocr successful for file with dossierId {} and fileId {}, took {}s", dossierId, fileId, format("%.1f", (ocrEnd - ocrStart) / 1000.0));
}
} finally {
fileStream.close();
transferOutputStream.close();
}
}
@SneakyThrows
private void runOcr(InputStream fileStream, OutputStream out, String fileId) {
PDFDoc pdfDoc = new PDFDoc(fileStream);
Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, true);
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).build());
// Optimization:
// When a page does not have a TextZone, PDFTron whites out the page. But, PDFTron scans it anyway, resulting in a longer runtime.
// So, we need to remove pages without images.
// Furthermore, creating a new document is *much* faster than reusing the same document and adding/removing pages one by one.
// Therefore, we create a new Document with a single page for every page that contains an image.
// For some reason, if we insert the OCR singlePageDoc into the original PDFDoc inside the loop, for some documents, the RAM Usage increases exponentially with every page.
// This is why, we replace the OCRed Pages outside the main loop.
int numProcessedPages = 0;
Map<Integer, PDFDoc> pageIdToSinglePagePdfDoc = new HashMap<>();
for (Integer pageId : pageIdToRectCollection.keySet()) {
try {
PDFDoc singlePagePdfDoc = extractSinglePagePdfDoc(pdfDoc, pageId);
processOcr(pageIdToRectCollection, pageId, singlePagePdfDoc);
log.info("{}/{} Page {} done, OCR regions {}",
numProcessedPages,
pageIdToRectCollection.size(),
pageId,
getAllOcrTextZonesAsString(pageIdToRectCollection, pageId));
pageIdToSinglePagePdfDoc.put(pageId, singlePagePdfDoc);
++numProcessedPages;
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).build());
} catch (PDFNetException e) {
log.error("Failed to process page {}", pageId);
throw new RuntimeException(e);
}
}
log.info("Copying {} OCRed Pages into original Document", pageIdToSinglePagePdfDoc.size());
pageIdToSinglePagePdfDoc.forEach((pageId, singlePagePdfDoc) -> replaceOriginalPageWithOcrPage(pdfDoc, pageId, singlePagePdfDoc));
Optimizer.optimize(pdfDoc);
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(pageIdToRectCollection.size()).numberOfOCRedPages(numProcessedPages).ocrFinished(true).build());
try {
pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
} catch (Exception e) {
log.error("Processed File with fileId {} could not be saved", fileId);
throw new RuntimeException(e);
}
pdfDoc.close();
}
private void processOcr(Map<Integer, RectCollection> pageIdToRectCollection, Integer pageId, PDFDoc singlePagePdfDoc) throws PDFNetException {
OCROptions options = new OCROptions();
options.addTextZonesForPage(pageIdToRectCollection.get(pageId), 1);
options.addLang(ENGLISH);
options.addDPI(settings.getOcrDPI());
OCRModule.processPDF(singlePagePdfDoc, options);
}
private static PDFDoc extractSinglePagePdfDoc(PDFDoc pdfDoc, Integer pageId) throws PDFNetException {
PDFDoc singlePagePdfDoc = new PDFDoc();
Page page = pdfDoc.getPage(pageId);
page.setMediaBox(page.getCropBox()); // this line ensures the ocr text is placed correctly by PDFTron, see TestFile MediaBoxBiggerThanCropBox.pdf
singlePagePdfDoc.pagePushBack(page);
return singlePagePdfDoc;
}
@SneakyThrows
private static void replaceOriginalPageWithOcrPage(PDFDoc pdfDoc, Integer pageId, PDFDoc singlePagePdfDoc) {
Page ocrPage = singlePagePdfDoc.getPage(1);
pdfDoc.pageInsert(pdfDoc.getPageIterator(pageId), ocrPage);
pdfDoc.pageRemove(pdfDoc.getPageIterator(pageId + 1));
singlePagePdfDoc.close();
}
private static StringBuilder getAllOcrTextZonesAsString(Map<Integer, RectCollection> pageIdToRectCollection, Integer pageId) throws PDFNetException {
StringBuilder zonesString = new StringBuilder();
for (int j = 0; j < pageIdToRectCollection.get(pageId).getNumRects(); ++j) {
var r = pageIdToRectCollection.get(pageId).getRectAt(j);
zonesString.append(format("[lower left (%.1f|%.1f) upper right (%.1f|%.1f)]", r.getX1(), r.getY1(), r.getX2(), r.getY2()));
}
return zonesString;
}
}
@@ -1,125 +0,0 @@
package com.iqser.red.service.ocr.v1.server.service;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Set;
import java.util.TreeSet;
import org.springframework.stereotype.Service;
import com.pdftron.pdf.Element;
import com.pdftron.pdf.ElementReader;
import com.pdftron.pdf.ElementWriter;
import com.pdftron.pdf.PDFDoc;
import com.pdftron.pdf.Page;
import com.pdftron.pdf.PageIterator;
import com.pdftron.sdf.Obj;
import com.pdftron.sdf.SDFDoc;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class WatermarkRemovalService {
/**
* !!!Warning!! This logic is definitive wrong and should NEVER run in production,
* however it was used in second DocuMine (SCM) prototype and we currently need it to compare the results.
*
* @param pdfFile the file as Inputstream.
* @param transferOutputStream the resulting file as Outputstream.
*/
@SneakyThrows
public void removeWatermarks(InputStream pdfFile, OutputStream transferOutputStream) {
PDFDoc pdfDoc = new PDFDoc(pdfFile);
this.execute(pdfDoc);
try {
pdfDoc.save(transferOutputStream, SDFDoc.SaveMode.LINEARIZED, null);
} catch (Exception var10) {
log.error("File could not be saved after watermark removal");
throw new RuntimeException(var10);
} finally {
pdfDoc.close();
}
}
@SneakyThrows
private void execute(PDFDoc pdfDoc) {
ElementWriter writer = new ElementWriter();
ElementReader reader = new ElementReader();
Set<Integer> visited = new TreeSet<>();
for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
Page page = iterator.next();
removeOverlapText(page, reader, writer, visited);
}
reader.destroy();
writer.destroy();
}
@SneakyThrows
private void removeOverlapText(Page page, ElementReader reader, ElementWriter writer, Set<Integer> visited) {
visited.add((int) page.getSDFObj().getObjNum());
reader.begin(page);
writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
processElements(reader, writer, visited, false);
writer.end();
reader.end();
}
@SneakyThrows
private void processElements(ElementReader reader, ElementWriter writer, Set<Integer> visited, boolean isInForm) {
for (Element element = reader.next(); element != null; element = reader.next())
switch (element.getType()) {
case Element.e_image, Element.e_inline_image -> processImage(element, writer, isInForm);
case Element.e_form -> processForm(reader, writer, element, visited);
default -> writer.writeElement(element);
}
}
@SneakyThrows
private void processForm(ElementReader reader, ElementWriter writer, Element element, Set<Integer> visited) {
writer.writeElement(element);
Obj formObj = element.getXObject();
if (!visited.contains((int) formObj.getObjNum())) {
visited.add((int) formObj.getObjNum());
ElementWriter formWriter = new ElementWriter();
reader.formBegin();
formWriter.begin(formObj);
reader.clearChangeList();
formWriter.setDefaultGState(reader);
processElements(reader, formWriter, visited, true);
formWriter.end();
formWriter.destroy();
reader.end();
}
}
@SneakyThrows
private void processImage(Element element, ElementWriter writer, boolean isInForm) {
// !!! Warning, this will also remove none watermark images form files.
// Idea: Remove watermarks by comparing (hash values) images. Watermarks to remove should be uploaded in dossier/dossierTemplate.
// Removing watermarks should be done in preprocessing, not at ocr.
if (!isInForm) {
writer.writeElement(element);
}
}
}
@@ -1,14 +0,0 @@
package com.iqser.red.service.ocr.v1.server.settings;
import org.springframework.boot.context.properties.ConfigurationProperties;
import lombok.Data;
@Data
@ConfigurationProperties("ocr-service")
public class OcrServiceSettings {
private int ocrDPI = 300;
private boolean removeWatermark;
}
@@ -1,133 +0,0 @@
package com.iqser.red.service.ocr.v1.server;
import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.ocr.v1.server.service.FileStorageService;
import com.iqser.red.service.ocr.v1.server.service.OCRService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.pdftron.pdf.OCRModule;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.micrometer.prometheus.PrometheusTimer;
import lombok.SneakyThrows;
@Disabled // OCRModule is not available on build server. If you want to run the test set the property at the top.
@SpringBootTest(properties = {"pdftron.ocrmodule.path=/YourOCRModulePath"})
public class OcrServiceIntegrationTest extends AbstractTest {
@Autowired
protected ObjectMapper objectMapper;
@Autowired
private OCRService ocrService;
@Autowired
private PrometheusMeterRegistry registry;
@BeforeEach
@SneakyThrows
public void assertOCRModuleIsLoaded() {
assert OCRModule.isModuleAvailable();
}
@Test
public void testOCRMetrics() {
testOCR("Watermark");
testOCR("Watermark");
testOCR("Watermark");
var ocrOnDocumentMeter = registry.getMeters().stream().filter(m -> m.getId().getName().equalsIgnoreCase("redactmanager_runOcrOnDocument")).findAny();
assertThat(ocrOnDocumentMeter.isPresent()).isTrue();
PrometheusTimer timer = (PrometheusTimer) ocrOnDocumentMeter.get();
assertThat(timer.count()).isEqualTo(3);
assertThat(timer.mean(TimeUnit.SECONDS)).isGreaterThan(0.1);
}
@Test
public void testOcr() {
// check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
String text = testOCR("StitchedImagesMultiPage");
}
@Test
public void testManyRotatedImages() {
// check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
String text = testOCR("manyRotatedImages");
assertThat(text).contains("Michela", "Gregori", "DVM", "PhD", "Pathologist");
}
@Test
public void testMergeImages() {
// check visually for most of the images containing text, the resulting text is kind of nonsense, just ensure it is there
String text = testOCR("merge_images");
assertThat(text).contains("Bodyweight change of dams with live young - group mean values",
"Control",
"mg/g day",
"10 mg/kg/day",
"20 mg/kg/",
"Days",
"50",
"-200",
"—250",
"150",
"200",
"250",
"—150");
}
@Test
public void testOCRWatermark() {
assertThat(testOCR("Watermark")).contains("syngenta");
}
@Test
public void testOCRInvisibleText() {
String text = testOCR("InvisibleText");
assertThat(text).contains("Michela", "Gregori", "DVM", "PhD", "Pathologist", "SIGNATURE PAGE");
assertThat(text).doesNotContain("COMPLETION DATE:", "LABORATORY PROJECT ID:", "AUTHOR(S):", "Substance");
}
@SneakyThrows
private String testOCR(String fileName) {
ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
var originId = FileStorageService.getStorageId("dossier", "file", FileType.ORIGIN);
try (var fileStream = pdfFileResource.getInputStream()) {
storageService.storeObject(TenantContext.getTenantId(), originId, fileStream);
}
try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
ocrService.runOcrOnDocument("dossier", "file", out);
System.out.println("File:" + getTemporaryDirectory() + "/" + fileName + ".pdf");
}
try (var fileStream = new FileInputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
return extractAllTextFromDocument(fileStream);
}
}
}
@@ -1,165 +0,0 @@
package com.iqser.red.service.ocr.v1.server.service;
import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawGrid;
import static com.iqser.red.service.ocr.v1.server.utils.PdfDraw.drawRectCollection;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.ocr.v1.server.AbstractTest;
import com.pdftron.common.PDFNetException;
import com.pdftron.pdf.ElementWriter;
import com.pdftron.pdf.PDFDoc;
import com.pdftron.pdf.RectCollection;
import com.pdftron.sdf.SDFDoc;
import lombok.SneakyThrows;
class ImagePositionRetrievalServiceTest extends AbstractTest {
@Autowired
private ImagePositionRetrievalService imagePositionRetrievalService;
@Test
@SneakyThrows
public void testImagePositionRetrievalForRotateTestFileWithImages() {
String fileName = "RotateTestFileWithImages";
List<int[]> allRectCoords = testImagePositionDetection(fileName);
assertThat(allRectCoords).contains(new int[]{48, 572, 295, 721},
new int[]{54, 279, 301, 428},
new int[]{360, 173, 509, 419},
new int[]{362, 522, 511, 768},
new int[]{459, 354, 608, 600},
new int[]{145, 404, 392, 553},
new int[]{151, 111, 398, 260},
new int[]{457, 5, 606, 251},
new int[]{395, 480, 545, 726},
new int[]{393, 130, 542, 377},
new int[]{88, 236, 334, 386},
new int[]{82, 530, 328, 679},
new int[]{465, 11, 614, 257},
new int[]{159, 117, 406, 266},
new int[]{467, 360, 617, 607},
new int[]{153, 410, 400, 559});
}
@Test
@SneakyThrows
public void testImagePositionRetrievalForRotateTestFileWithImagesExtremeCropbox() {
String fileName = "RotateTestFileWithImagesExtremeCropbox";
List<int[]> allRectCoords = testImagePositionDetection(fileName);
assertThat(allRectCoords).contains(new int[]{48, 572, 295, 721},
new int[]{362, 522, 511, 768},
new int[]{360, 173, 509, 419},
new int[]{54, 279, 301, 428},
new int[]{145, 192, 392, 341},
new int[]{459, 142, 608, 388},
new int[]{457, -207, 606, 39},
new int[]{151, -101, 398, 48},
new int[]{-30, 238, 216, 387},
new int[]{283, 188, 433, 434},
new int[]{281, -162, 430, 85},
new int[]{-24, -56, 222, 94},
new int[]{-39, 410, 208, 559},
new int[]{275, 360, 425, 607},
new int[]{273, 11, 422, 257},
new int[]{-33, 117, 214, 266});
}
@Test
@SneakyThrows
public void testMergeImages() {
String fileName = "merge_images";
List<int[]> allRectCoords = testImagePositionDetection(fileName);
assertThat(allRectCoords).contains(new int[]{90, 284, 398, 770});
}
@Test
@SneakyThrows
public void testStitchedImagesMultiPage() {
String fileName = "StitchedImagesMultiPage";
List<int[]> allRectCoords = testImagePositionDetection(fileName);
assertThat(allRectCoords.size()).isEqualTo(48);
}
@Test
@SneakyThrows
public void testEveryPointInDashedLineIsImage() {
String fileName = "everyPointInDashedLineIsImage";
List<int[]> allRectCoords = testImagePositionDetection(fileName);
assertThat(allRectCoords.size()).isEqualTo(0);
}
private List<int[]> testImagePositionDetection(String fileName) throws IOException, PDFNetException {
try (InputStream fileStream = new FileInputStream(new ClassPathResource("files/" + fileName + ".pdf").getFile().getAbsolutePath())) {
PDFDoc pdfDoc = new PDFDoc(fileStream);
Map<Integer, RectCollection> pageIdToRectCollection = imagePositionRetrievalService.getImagePositionPerPage(pdfDoc, false);
ElementWriter writer = new ElementWriter();
pageIdToRectCollection.forEach((pageId, rectCollection) -> {
try {
writer.begin(pdfDoc.getPage(pageId));
drawRectCollection(writer, rectCollection);
drawGrid(writer, pdfDoc.getPage(pageId));
writer.end();
StringBuilder zonesString = new StringBuilder();
for (int j = 0; j < rectCollection.getNumRects(); ++j) {
var r = rectCollection.getRectAt(j);
zonesString.append(String.format("%d:[lower left (%.1f|%.1f) upper right (%.1f|%.1f)]; ", j, r.getX1(), r.getY1(), r.getX2(), r.getY2()));
}
System.out.printf("Page %d: rotation: %d OCR regions %s \n", pageId, pdfDoc.getPage(pageId).getRotation(), zonesString);
} catch (PDFNetException e) {
throw new RuntimeException(e);
}
});
// Check visually for red Rectangles to match images in the saved pdf file
try (var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf")) {
out.write(pdfDoc.save(SDFDoc.SaveMode.LINEARIZED, null));
}
pdfDoc.close();
System.out.println("result file: " + getTemporaryDirectory() + "/" + fileName + "_IMAGE_BBOX.pdf");
// round all coords to nearest int to account for inconsistencies with the calculation of the bounding box
return pageIdToRectCollection.values().stream().map(this::toRoundedCoordinateArrayList).flatMap(List::stream).collect(Collectors.toList());
}
}
@SneakyThrows
private List<int[]> toRoundedCoordinateArrayList(RectCollection rectCollection) {
List<int[]> coords = new ArrayList<>(rectCollection.getNumRects());
for (int i = 0; i < rectCollection.getNumRects(); ++i) {
var r = rectCollection.getRectAt(i);
coords.add(new int[]{(int) Math.round(r.getX1()), (int) Math.round(r.getY1()), (int) Math.round(r.getX2()), (int) Math.round(r.getY2())});
}
return coords;
}
}
@@ -1,51 +0,0 @@
package com.iqser.red.service.ocr.v1.server.service;
import static com.iqser.red.pdftronlogic.commons.PdfTextExtraction.extractAllTextFromDocument;
import static com.iqser.red.service.ocr.v1.server.utils.OsUtils.getTemporaryDirectory;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
import com.iqser.red.service.ocr.v1.server.AbstractTest;
import lombok.SneakyThrows;
public class InvisibleElementRemovalServiceTest extends AbstractTest {
@Autowired
private InvisibleElementRemovalService invisibleElementRemovalService;
@Test
@SneakyThrows
public void testRemoveInvisibleText() {
String fileName = "InvisibleText";
ClassPathResource pdfFileResource = new ClassPathResource("files/" + fileName + ".pdf");
try (var initialFileStream = pdfFileResource.getInputStream(); var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out, false);
}
try (var initialFileStream = pdfFileResource.getInputStream(); var out = new FileOutputStream(getTemporaryDirectory() + "/" + fileName + "_delta.pdf")) {
invisibleElementRemovalService.removeInvisibleElements(initialFileStream, out, true);
}
System.out.println("Output File without invisible elements: files/" + fileName + ".pdf");
System.out.println("Output Delta File without invisible elements: files/" + fileName + "_delta.pdf");
try (var fileStream = new FileInputStream(getTemporaryDirectory() + "/" + fileName + ".pdf")) {
String[] text = extractAllTextFromDocument(fileStream).split("\n");
assertThat(text).contains("APPENDIX 16 Pathology Report", "Amendment 1", "Page 255 of 260");
}
}
}
@@ -1,119 +0,0 @@
package com.iqser.red.service.ocr.v1.server.utils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
public class FileSystemBackedStorageService implements StorageService {
private final Map<String, File> dataMap = new HashMap<>();
public FileSystemBackedStorageService() {
}
@SneakyThrows
@Override
public InputStreamResource getObject(String tenantId, String objectId) {
var res = dataMap.get(objectId);
if (res == null) {
throw new StorageObjectDoesNotExist(new RuntimeException());
}
return new InputStreamResource(new FileInputStream(res));
}
@Override
public void deleteObject(String tenantId, String objectId) {
dataMap.remove(objectId);
}
@Override
public boolean objectExists(String tenantId, String objectId) {
return dataMap.containsKey(objectId);
}
@Override
@SneakyThrows
public <T> void storeJSONObject(String tenantId, String objectId, T any) {
File tempFile = File.createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any);
dataMap.put(objectId, tempFile);
}
private ObjectMapper getMapper() {
return ObjectMapperFactory.create();
}
@Override
@SneakyThrows
public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
throw new StorageObjectDoesNotExist("Stored object not found");
}
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
}
public List<String> listPaths() {
return new ArrayList<>(dataMap.keySet());
}
public List<String> listFilePaths() {
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
}
@Override
@SneakyThrows
public void storeObject(String tenantId, String objectId, InputStream stream) {
File tempFile = File.createTempFile("test", ".tmp");
try (var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.copy(stream, fileOutputStream);
}
dataMap.put(objectId, tempFile);
}
public void clearStorage() {
this.dataMap.forEach((k, v) -> v.delete());
this.dataMap.clear();
}
}
@@ -0,0 +1,82 @@
import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
application
id("com.iqser.red.service.java-conventions")
id("org.springframework.boot") version "3.1.5"
id("io.spring.dependency-management") version "1.1.3"
id("org.sonarqube") version "4.3.0.3225"
id("io.freefair.lombok") version "8.2.2"
}
configurations {
all {
exclude(group = "commons-logging", module = "commons-logging")
exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
exclude(group = "com.iqser.red.commons", module = "logging-commons")
}
}
val springBootStarterVersion = "3.1.5"
dependencies {
implementation(project(":ocr-service-processor"))
implementation(project(":ocr-service-api"))
implementation("com.knecon.fforesight:tracing-commons:0.3.0")
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
implementation("net.logstash.logback:logstash-logback-encoder:7.4")
implementation("ch.qos.logback:logback-classic")
testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
testImplementation("com.iqser.red.commons:test-commons:2.1.0")
testImplementation("org.springframework.amqp:spring-rabbit-test:3.0.2")
}
tasks.named<BootBuildImage>("bootBuildImage") {
environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
environment.put("BPE_GS_LIB", "/layers/fagiani_apt/apt/usr/share/ghostscript/9.26/Resource/Init/") // set ghostscript lib path
environment.put("BPE_FONTCONFIG_PATH", "/layers/fagiani_apt/apt/etc/fonts/") // set ghostscript fontconfig path
var aptfile = layout.projectDirectory.file("src/main/resources/Aptfile").toString()
bindings.add("${aptfile}:/workspace/Aptfile:ro")
val vcpkgFile = layout.projectDirectory.file("src/main/resources/vcpkg.json").toString()
bindings.add("${vcpkgFile}:/workspace/vcpkg.json:ro")
val languagesFile = layout.projectDirectory.file("src/main/resources/tesseract_languages").toString()
bindings.add("${languagesFile}:/workspace/tesseract_languages:ro")
buildpacks.set(
listOf(
"ghcr.io/fagiani/buildpacks/fagiani_apt@sha256:6471c8c70f32b749e29f65ae562ac0339fecad26aa9217628c00a6c31f197dae",
"ghcr.io/kschuettler/knecon-vcpkg@sha256:ba5e967b124de4865ff7e8f565684f752dd6e97b302e2dcf651283f6a19b98b9",
"ghcr.io/kschuettler/knecon-tessdata@sha256:9062f728aa0340ac963bcdd6f5e740d683823a81d3f480db894da15bff72691a",
"urn:cnb:builder:paketo-buildpacks/java"
)
)
imageName.set("nexus.knecon.com:5001/ff/${project.name}") // must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
if (project.hasProperty("buildbootDockerHostNetwork")) {
network.set("host")
}
docker {
if (project.hasProperty("buildbootDockerHostNetwork")) {
bindHostToBuilder.set(true)
}
verboseLogging.set(true)
publishRegistry {
username.set(providers.gradleProperty("mavenUser").getOrNull())
password.set(providers.gradleProperty("mavenPassword").getOrNull())
email.set(providers.gradleProperty("mavenEmail").getOrNull())
url.set("https://nexus.knecon.com:5001/")
}
val dockerTag = "nexus.knecon.com:5001/ff/${project.name}:${project.version}"
tags.set(listOf(dockerTag))
}
}
@@ -1,20 +1,20 @@
package com.iqser.red.service.ocr.v1.server;
package com.knecon.fforesight.service.ocr.v1.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.scheduling.annotation.EnableAsync;
import com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService;
import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.ocr.v1.server.settings.OcrServiceSettings;
import com.iqser.red.pdftronlogic.commons.WatermarkRemovalService;
import com.knecon.fforesight.service.ocr.processor.OcrServiceProcessorConfiguration;
import com.knecon.fforesight.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
import com.knecon.fforesight.service.ocr.v1.server.queue.MessagingConfiguration;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@@ -23,9 +23,8 @@ import io.micrometer.core.instrument.MeterRegistry;
@EnableAsync
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@EnableConfigurationProperties(OcrServiceSettings.class)
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
@Import({ MessagingConfiguration.class, StorageAutoConfiguration.class})
@Import({MessagingConfiguration.class, StorageAutoConfiguration.class, OcrServiceProcessorConfiguration.class})
@EnableFeignClients(basePackageClasses = FileStatusProcessingUpdateClient.class)
public class Application {
@@ -35,7 +34,7 @@ public class Application {
* @param args Any command line parameter given upon startup.
*/
public static void main(String[] args) {
System.setProperty("org.apache.pdfbox.rendering.UsePureJavaCMYKConversion", "true");
SpringApplication.run(Application.class, args);
}
@@ -53,4 +52,11 @@ public class Application {
return new InvisibleElementRemovalService();
}
@Bean
public WatermarkRemovalService watermarkRemovalService() {
return new WatermarkRemovalService();
}
}
@@ -1,4 +1,4 @@
package com.iqser.red.service.ocr.v1.server.client;
package com.knecon.fforesight.service.ocr.v1.server.client;
import org.springframework.cloud.openfeign.FeignClient;
@@ -1,4 +1,4 @@
package com.iqser.red.service.ocr.v1.server.configuration;
package com.knecon.fforesight.service.ocr.v1.server.queue;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
@@ -1,7 +1,4 @@
package com.iqser.red.service.ocr.v1.server.service;
import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_HEADER;
import static com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
package com.knecon.fforesight.service.ocr.v1.server.queue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -17,24 +14,28 @@ import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.ocr.v1.api.model.DocumentRequest;
import com.iqser.red.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.ocr.v1.server.configuration.MessagingConfiguration;
import com.knecon.fforesight.service.ocr.v1.server.client.FileStatusProcessingUpdateClient;
import com.knecon.fforesight.service.ocr.processor.service.FileStorageService;
import com.knecon.fforesight.service.ocr.processor.service.OCRService;
import com.knecon.fforesight.service.ocr.v1.api.model.DocumentRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
import feign.FeignException;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class OcrMessageReceiver {
private final FileStorageService fileStorageService;
private final ObjectMapper objectMapper;
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
private final OCRService ocrService;
FileStorageService fileStorageService;
ObjectMapper objectMapper;
FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
OCRService ocrService;
@RabbitHandler
@@ -42,7 +43,7 @@ public class OcrMessageReceiver {
public void receiveOcr(Message in) throws IOException {
DocumentRequest ocrRequestMessage = objectMapper.readValue(in.getBody(), DocumentRequest.class);
log.info("--------------------------------------------------------------------------");
log.info("Start ocr for file with dossierId {} and fileId {}", ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
try {
@@ -66,21 +67,21 @@ public class OcrMessageReceiver {
fileStatusProcessingUpdateClient.ocrSuccessful(ocrRequestMessage.getDossierId(), ocrRequestMessage.getFileId());
} catch (Exception e) {
log.warn("An exception occurred in ocr file stage: {}", e.getMessage());
in.getMessageProperties().getHeaders().put(X_ERROR_INFO_HEADER, e.getMessage());
in.getMessageProperties().getHeaders().put(X_ERROR_INFO_TIMESTAMP_HEADER, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
throw e;
in.getMessageProperties().getHeaders().put(MessagingConfiguration.X_ERROR_INFO_HEADER, e.getMessage());
in.getMessageProperties().getHeaders().put(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER, OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
throw new RuntimeException(e);
}
}
@RabbitHandler
@RabbitListener(queues = MessagingConfiguration.OCR_DLQ, concurrency = "1")
public void receiveOcrDQL(Message failedMessage) throws IOException {
public void receiveOcrDLQ(Message failedMessage) throws IOException {
DocumentRequest ocrRequestMessage = objectMapper.readValue(failedMessage.getBody(), DocumentRequest.class);
log.info("OCR DQL received: {}", ocrRequestMessage);
String errorMessage = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_HEADER);
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(X_ERROR_INFO_TIMESTAMP_HEADER);
String errorMessage = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_HEADER);
OffsetDateTime timestamp = failedMessage.getMessageProperties().getHeader(MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER);
timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
fileStatusProcessingUpdateClient.ocrFailed(ocrRequestMessage.getDossierId(),
ocrRequestMessage.getFileId(),
@@ -0,0 +1,35 @@
package com.knecon.fforesight.service.ocr.v1.server.queue;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.ocr.processor.service.IOcrMessageSender;
import com.knecon.fforesight.service.ocr.v1.api.model.OCRStatusUpdateResponse;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class OcrMessageSender implements IOcrMessageSender {
RabbitTemplate rabbitTemplate;
public void sendOcrFinished(String fileId, int totalImages) {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(totalImages).numberOfOCRedPages(totalImages).ocrFinished(true).build());
}
public void sendUpdate(String fileId, int finishedImages, int totalImages) {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_STATUS_UPDATE_RESPONSE_QUEUE,
OCRStatusUpdateResponse.builder().fileId(fileId).numberOfPagesToOCR(totalImages).numberOfOCRedPages(finishedImages).build());
}
}

Some files were not shown because too many files have changed in this diff Show More