Compare commits

...
8 Commits
108 changed files with 797 additions and 866 deletions
+1
View File
@@ -18,6 +18,7 @@ target/
.settings
.springBeans
.sts4-cache
.gradle
### IntelliJ IDEA ###
.idea
+1 -1
View File
@@ -1,4 +1,4 @@
include:
- project: 'gitlab/gitlab'
ref: 'main'
file: 'ci-templates/maven_java.yml'
file: 'ci-templates/gradle_java.yml'
BIN
View File
Binary file not shown.
-18
View File
@@ -1,18 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.7/apache-maven-3.8.7-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar
+7
View File
@@ -0,0 +1,7 @@
plugins {
`kotlin-dsl`
}
repositories {
gradlePluginPortal()
}
@@ -0,0 +1,56 @@
plugins {
java
pmd
checkstyle
jacoco
}
group = "com.knecon.fforesight"
version = "0.1-SNAPSHOT"
java.sourceCompatibility = JavaVersion.VERSION_17
java.targetCompatibility = JavaVersion.VERSION_17
tasks.jacocoTestReport {
reports {
xml.required.set(false)
csv.required.set(false)
html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
}
}
tasks.pmdMain {
pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
}
tasks.pmdTest {
pmd.ruleSetFiles = files("${rootDir}/config/pmd/test_pmd.xml")
}
tasks.named<Test>("test") {
useJUnitPlatform()
}
tasks.test {
finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
}
tasks.jacocoTestReport {
dependsOn(tasks.test) // tests are required to run before generating the report
reports {
xml.required.set(true)
csv.required.set(false)
}
}
repositories {
mavenLocal()
mavenCentral()
maven {
url = uri("https://nexus.knecon.com/repository/gindev/");
credentials {
username = providers.gradleProperty("mavenUser").getOrNull();
password = providers.gradleProperty("mavenPassword").getOrNull();
}
}
}
+39
View File
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<module name="Checker">
<property
name="severity"
value="error"/>
<module name="TreeWalker">
<module name="SuppressWarningsHolder"/>
<module name="MissingDeprecated"/>
<module name="MissingOverride"/>
<module name="AnnotationLocation"/>
<module name="JavadocStyle"/>
<module name="NonEmptyAtclauseDescription"/>
<module name="IllegalImport"/>
<module name="RedundantImport"/>
<module name="RedundantModifier"/>
<module name="EmptyBlock"/>
<module name="DefaultComesLast"/>
<module name="EmptyStatement"/>
<module name="EqualsHashCode"/>
<module name="ExplicitInitialization"/>
<module name="IllegalInstantiation"/>
<module name="ModifiedControlVariable"/>
<module name="MultipleVariableDeclarations"/>
<module name="PackageDeclaration"/>
<module name="ParameterAssignment"/>
<module name="SimplifyBooleanExpression"/>
<module name="SimplifyBooleanReturn"/>
<module name="StringLiteralEquality"/>
<module name="OneStatementPerLine"/>
<module name="FinalClass"/>
<module name="ArrayTypeStyle"/>
<module name="UpperEll"/>
<module name="OuterTypeFilename"/>
</module>
<module name="FileTabCharacter"/>
<module name="SuppressWarningsFilter"/>
</module>
+21
View File
@@ -0,0 +1,21 @@
<?xml version="1.0"?>
<ruleset name="Custom ruleset"
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
<description>
Knecon ruleset checks the code for bad stuff
</description>
<rule ref="category/java/errorprone.xml">
<exclude name="DataflowAnomalyAnalysis"/>
<exclude name="MissingSerialVersionUID"/>
<exclude name="NullAssignment"/>
<exclude name="BeanMembersShouldSerialize"/>
<exclude name="AvoidLiteralsInIfCondition"/>
<exclude name="AvoidDuplicateLiterals"/>
<exclude name="AvoidFieldNameMatchingMethodName"/>
</rule>
</ruleset>
+23
View File
@@ -0,0 +1,23 @@
<?xml version="1.0"?>
<ruleset name="Custom ruleset"
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
<description>
Knecon test ruleset checks the code for bad stuff
</description>
<rule ref="category/java/errorprone.xml">
<exclude name="DataflowAnomalyAnalysis"/>
<exclude name="MissingSerialVersionUID"/>
<exclude name="NullAssignment"/>
<exclude name="BeanMembersShouldSerialize"/>
<exclude name="AvoidLiteralsInIfCondition"/>
<exclude name="AvoidDuplicateLiterals"/>
<exclude name="AvoidFieldNameMatchingMethodName"/>
<exclude name="AvoidFieldNameMatchingTypeName"/>
</rule>
</ruleset>
+1
View File
@@ -0,0 +1 @@
version = 0.1.0
-99
View File
@@ -1,99 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.knecon.fforesight</groupId>
<artifactId>platform-docker-dependency</artifactId>
<version>0.1.0</version>
<relativePath/>
</parent>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service-image</artifactId>
<version>0.1-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<service.server>layoutparser-service-server</service.server>
<platform.jar>${service.server}.jar</platform.jar>
<docker.skip.push>false</docker.skip.push>
<docker.image.prefix>ff</docker.image.prefix>
<docker.image.name>${docker.image.prefix}/${service.server}</docker.image.name>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>download-platform-jar</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<artifactItems>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>${service.server}</artifactId>
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
<destFileName>${platform.jar}</destFileName>
</dependency>
</artifactItems>
<outputDirectory>${docker.build.directory}</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
<configuration>
<images>
<image>
<name>${docker.image.name}</name>
<build>
<dockerFileDir>${docker.build.directory}</dockerFileDir>
<args>
<PLATFORM_JAR>${platform.jar}</PLATFORM_JAR>
</args>
<tags>
<tag>${docker.image.version}</tag>
<tag>latest</tag>
</tags>
</build>
</image>
</images>
</configuration>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
@@ -1,9 +0,0 @@
FROM red/base-image:2.0.2
ARG PLATFORM_JAR
ENV PLATFORM_JAR ${PLATFORM_JAR}
ENV USES_ELASTICSEARCH false
COPY ["${PLATFORM_JAR}", "/"]
@@ -0,0 +1,6 @@
plugins {
id("com.knecon.fforesight.java-conventions")
id("io.freefair.lombok") version "8.1.0"
}
description = "layoutparser-service-internal-api"
@@ -1,24 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service</artifactId>
<version>0.1-SNAPSHOT</version>
</parent>
<artifactId>layoutparser-service-internal-api</artifactId>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
</dependencies>
</project>
@@ -17,5 +17,4 @@ public class DocumentData {
DocumentPositionData[] documentPositions;
DocumentStructure documentStructure;
}
@@ -4,12 +4,14 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentPage {
int number;
@@ -4,12 +4,14 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentPositionData {
Long id;
@@ -8,14 +8,13 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentStructure {
@@ -53,10 +52,11 @@ public class DocumentStructure {
}
@Data
@Builder
@Getter
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@FieldDefaults(level = AccessLevel.PRIVATE)
public static class EntryData {
NodeType type;
@@ -6,12 +6,14 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentTextData {
Long id;
@@ -22,6 +24,4 @@ public class DocumentTextData {
int end;
int[] lineBreaks;
}
@@ -12,4 +12,5 @@ public class TableData {
List<RowData> rowData;
Integer numberOfCols;
Integer numberOfRows;
}
@@ -0,0 +1,21 @@
plugins {
id("com.knecon.fforesight.java-conventions")
id("io.freefair.lombok") version "8.1.0"
}
dependencies {
implementation(project(":layoutparser-service-internal-api"))
implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
implementation("com.knecon.fforesight:tenant-commons:0.10.0")
implementation("com.iqser.red.commons:storage-commons:2.1.0")
implementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
implementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
}
description = "layoutparser-service-processor"
@@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service</artifactId>
<version>0.1-SNAPSHOT</version>
</parent>
<artifactId>layoutparser-service-processor</artifactId>
<dependencies>
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-shared-api-v1</artifactId>
<version>2.36.0</version>
</dependency>
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>tenant-commons</artifactId>
<version>${tennat-commons.version}</version>
</dependency>
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service-internal-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>storage-commons</artifactId>
<version>${storage-commons.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox-tools</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-afterburner</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>RELEASE</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
@@ -10,14 +10,14 @@ import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
@@ -48,40 +48,40 @@ public class LayoutParsingPipeline {
public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
long start = System.currentTimeMillis();
PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
}
TableServiceResponse tableServiceResponse = new TableServiceResponse();
if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
}
Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
int numberOfPages = originDocument.getNumberOfPages();
layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
}
return LayoutParsingFinishedEvent.builder()
.identifier(layoutParsingRequest.identifier())
.numberOfPages(numberOfPages)
.duration(System.currentTimeMillis() - start)
.message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
layoutParsingRequest.structureFileStorageId(),
layoutParsingRequest.textBlockFileStorageId(),
layoutParsingRequest.positionBlockFileStorageId(),
layoutParsingRequest.pageFileStorageId()))
.build();
}
TableServiceResponse tableServiceResponse = new TableServiceResponse();
if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
}
Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
int numberOfPages = originDocument.getNumberOfPages();
originDocument.close();
layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
}
return LayoutParsingFinishedEvent.builder()
.identifier(layoutParsingRequest.identifier())
.numberOfPages(numberOfPages)
.duration(System.currentTimeMillis() - start)
.message(format("Layout parsing is finished and files have been saved with Ids:\n Structure: %s\nText: %s\nPositions: %s\nPageData: %s",
layoutParsingRequest.structureFileStorageId(),
layoutParsingRequest.textBlockFileStorageId(),
layoutParsingRequest.positionBlockFileStorageId(),
layoutParsingRequest.pageFileStorageId()))
.build();
}
@@ -115,7 +115,8 @@ public class LayoutParsingPipeline {
long start = System.currentTimeMillis();
ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType, originDocument,
ClassificationDocument classificationDocument = pdfParsingService.parseDocument(layoutParsingType,
originDocument,
cvTableParsingAdapter.buildCvParsedTablesPerPage(tableServiceResponse),
imageServiceResponseAdapter.buildClassifiedImagesPerPage(imageServiceResponse));
@@ -22,8 +22,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
@@ -1,5 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
public interface GenericSemanticNode extends SemanticNode {
}
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph;
package com.knecon.fforesight.service.layoutparser.processor.model.graph;
import static java.lang.String.format;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph;
package com.knecon.fforesight.service.layoutparser.processor.model.graph;
import static java.lang.String.format;
@@ -8,13 +8,13 @@ import java.util.List;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
public enum EntityType {
ENTITY,
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
import java.awt.geom.Rectangle2D;
import java.util.Collection;
@@ -11,9 +11,9 @@ import java.util.Map;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
import lombok.AccessLevel;
@@ -1,9 +1,9 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
import java.awt.geom.Rectangle2D;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
@@ -12,10 +12,10 @@ import java.util.stream.Stream;
import com.amazonaws.services.kms.model.NotFoundException;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -0,0 +1,5 @@
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
public interface GenericSemanticNode extends SemanticNode {
}
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
@@ -9,10 +9,10 @@ import java.util.Map;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,6 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.Locale;
public enum ImageType {
LOGO,
@@ -10,7 +12,7 @@ public enum ImageType {
public static ImageType fromString(String imageType) {
return switch (imageType.toLowerCase()) {
return switch (imageType.toLowerCase(Locale.ROOT)) {
case "logo" -> ImageType.LOGO;
case "formula" -> ImageType.FORMULA;
case "signature" -> ImageType.SIGNATURE;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.LinkedList;
@@ -6,9 +6,9 @@ import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,14 +1,14 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import static java.lang.String.format;
@@ -6,18 +6,19 @@ import java.awt.geom.Rectangle2D;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.EntityType;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.EntityType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
public interface SemanticNode {
@@ -285,7 +286,7 @@ public interface SemanticNode {
*/
default boolean containsStringIgnoreCase(String string) {
return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
return getTextBlock().getSearchText().toLowerCase(Locale.ROOT).contains(string.toLowerCase(Locale.ROOT));
}
@@ -1,19 +1,20 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import static java.lang.String.format;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -67,7 +68,7 @@ public class Table implements SemanticNode {
*/
public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
}
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
@@ -8,10 +8,10 @@ import java.util.Map;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import static java.lang.String.format;
@@ -13,9 +13,9 @@ import java.util.Map;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
import lombok.AccessLevel;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import static java.lang.String.format;
@@ -11,8 +11,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import lombok.AccessLevel;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import static java.lang.String.format;
@@ -10,8 +10,8 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
public interface TextBlock extends CharSequence {
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import java.util.Set;
import java.util.function.BiConsumer;
@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.image;
import java.awt.geom.Rectangle2D;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
import lombok.Data;
import lombok.NonNull;
@@ -4,6 +4,7 @@ import java.awt.geom.AffineTransform;
import java.awt.geom.Point2D;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;
import org.apache.pdfbox.text.TextPosition;
@@ -76,7 +77,7 @@ public class TextPositionSequence implements CharSequence {
RedTextPosition textPosition = textPositionAt(index);
String text = textPosition.getUnicode();
return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
return caseInSensitive ? text.toLowerCase(Locale.ROOT).charAt(0) : text.charAt(0);
}
@@ -223,7 +224,7 @@ public class TextPositionSequence implements CharSequence {
@JsonAttribute(ignore = true)
public String getFont() {
return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
}
@@ -231,7 +232,7 @@ public class TextPositionSequence implements CharSequence {
@JsonAttribute(ignore = true)
public String getFontStyle() {
String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
return "bold, italic";
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter;
package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
import java.util.ArrayList;
import java.util.Collection;
@@ -8,8 +8,8 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter;
package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
@@ -9,10 +9,10 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
import lombok.RequiredArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import java.util.HashMap;
import java.util.Map;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import java.util.ArrayList;
import java.util.List;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.AllArgsConstructor;
import lombok.Builder;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.Builder;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import java.util.ArrayList;
import java.util.List;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import java.util.ArrayList;
import java.util.List;
@@ -6,6 +6,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.stream.Stream;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.knecon.fforesight.service.layoutparser.processor.model.GapInformation;
@@ -93,7 +94,7 @@ public class GapsAcrossLinesService {
int lineCount = 1;
public GapAcrossLines(Rectangle2D rectangle2D) {
GapAcrossLines(Rectangle2D rectangle2D) {
this.rectangle2D = correctRectangle(rectangle2D);
}
@@ -10,8 +10,8 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -19,7 +19,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.image.Classifi
import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
@@ -12,7 +12,7 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
@@ -20,11 +20,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import lombok.RequiredArgsConstructor;
@@ -15,7 +15,7 @@ import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
import lombok.experimental.UtilityClass;
@@ -28,31 +28,30 @@ public class TextPositionSequenceSorter {
List<PageContents> textPositionSequencesPerPage = new LinkedList<>();
try (InputStream inputStream = new ClassPathResource(filename).getInputStream()) {
PDDocument pdDocument = Loader.loadPDF(inputStream);
try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
for (int pageNumber = 1; pageNumber < pdDocument.getNumberOfPages() + 1; pageNumber++) {
PDFLinesTextStripper stripper = new PDFLinesTextStripper();
PDPage pdPage = pdDocument.getPage(pageNumber - 1);
stripper.setPageNumber(pageNumber);
stripper.setSortByPosition(true);
stripper.setStartPage(pageNumber);
stripper.setEndPage(pageNumber);
stripper.setPdpage(pdPage);
stripper.getText(pdDocument);
PDFLinesTextStripper stripper = new PDFLinesTextStripper();
PDPage pdPage = pdDocument.getPage(pageNumber - 1);
stripper.setPageNumber(pageNumber);
stripper.setSortByPosition(true);
stripper.setStartPage(pageNumber);
stripper.setEndPage(pageNumber);
stripper.setPdpage(pdPage);
stripper.getText(pdDocument);
Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
.stream()
.collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
Map<Float, List<TextPositionSequence>> sortedTextPositionSequencesPerDir = stripper.getTextPositionSequences()
.stream()
.collect(Collectors.groupingBy(textPositionSequence -> textPositionSequence.getDir().getDegrees()));
var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());
var sortedTextPositionSequences = sortByDirAccordingToPageRotation(sortedTextPositionSequencesPerDir, pdPage.getRotation());
textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
textPositionSequencesPerPage.add(new PageContents(sortedTextPositionSequences,
RectangleTransformations.toRectangle2D(pdPage.getCropBox()),
RectangleTransformations.toRectangle2D(pdPage.getMediaBox())));
}
}
pdDocument.close();
}
return textPositionSequencesPerPage;
@@ -39,7 +39,11 @@ public class DocuMineBlockificationService {
List<TextPositionSequence> chunkWords = new ArrayList<>();
List<AbstractPageBlock> chunkBlockList1 = new ArrayList<>();
float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
float minX = 1000;
float maxX = 0;
float minY = 1000;
float maxY = 0;
TextPositionSequence prev = null;
boolean wasSplitted = false;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy;
@@ -20,17 +20,17 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
@@ -1,10 +1,10 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import lombok.AccessLevel;
import lombok.Builder;
@@ -1,16 +1,17 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import java.awt.geom.AffineTransform;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import lombok.experimental.UtilityClass;
@@ -132,7 +133,7 @@ public class SearchTextWithTextPositionFactory {
private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {
if (currentTextPosition.getFontName().toLowerCase().contains(fontType)) {
if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
fontTypePositions.add(stringIdx);
}
}
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
@@ -14,9 +14,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;
import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import static java.util.Collections.emptyList;
@@ -11,12 +11,12 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
import lombok.experimental.UtilityClass;
@@ -1,11 +1,11 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
@@ -12,14 +12,14 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.util.Arrays;
import java.util.HashSet;
@@ -12,21 +12,21 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.awt.geom.Rectangle2D;
import java.util.Arrays;
@@ -6,10 +6,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
public class PropertiesMapper {
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.taas;
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.awt.geom.Rectangle2D;
import java.util.List;
@@ -13,13 +13,13 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Researc
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.RowData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.StructureObject;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.TableData;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
public class TaasDocumentDataMapper {
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.knecon.fforesight.service.layoutparser.processor.parsing;
package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
import java.io.IOException;
import java.io.InputStream;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.parsing;
package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
import java.awt.color.CMMException;
import java.awt.geom.Point2D;
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.knecon.fforesight.service.layoutparser.processor.parsing;
package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
import java.io.BufferedInputStream;
import java.io.IOException;
@@ -9,7 +9,7 @@ import java.util.stream.Collectors;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import lombok.experimental.UtilityClass;
@@ -15,11 +15,11 @@ import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.Builder;
@@ -18,7 +18,7 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
@@ -0,0 +1,102 @@
import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
id("com.knecon.fforesight.java-conventions")
id("org.springframework.boot") version "3.1.2"
id("io.spring.dependency-management") version "1.1.0"
id("org.sonarqube") version "4.2.1.3168"
id("io.freefair.lombok") version "8.1.0"
//id("org.graalvm.buildtools.native") version "0.9.23"
}
dependencies {
implementation(project(":layoutparser-service-processor"))
implementation("com.iqser.red.commons:storage-commons:2.1.0")
implementation("com.knecon.fforesight:tenant-commons:0.10.0")
implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
// for integration testing only
testImplementation(project(":layoutparser-service-internal-api"))
testImplementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
testImplementation("com.iqser.red.commons:jackson-commons:1.0.0")
testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
}
description = "layoutparser-service-server"
java {
withJavadocJar()
}
/*
TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something
kinda weird behavior, since the buildArgs below seem to have no effect at all
https://stackoverflow.com/questions/72770461/graalvm-native-image-can-not-compile-logback-dependencies
// AOT seems to be the name of the generated classes for native images
// They are added as a SourceSet, and therefore checkstyle and pmd try to run on it
tasks.named<Checkstyle>("checkstyleAot") {
enabled = false
}
tasks.named<Checkstyle>("checkstyleAotTest") {
enabled = false
}
tasks.named<Pmd>("pmdAot") {
enabled = false
}
tasks.named<Pmd>("pmdAotTest") {
enabled = false
}
graalvmNative {
binaries {
all {
resources.autodetect()
buildArgs.add("--verbose")
buildArgs.add("--add-opens=java.base/java.nio=ALL-UNNAMED")
buildArgs.add("--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED")
buildArgs.add("--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED")
buildArgs.add("--trace-class-initialization=ch.qos.logback.classic.Logger")
buildArgs.add("--trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase\$Worker")
buildArgs.add("--initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback")
buildArgs.add("--initialize-at-run-time=io.netty")
}
}
}
nativeBuild {
buildArgs("-H:ReflectionConfigurationFiles=../../../src/main/resources/reflection-config.json")
}
*/
tasks.named<BootBuildImage>("bootBuildImage") {
imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
if (project.hasProperty("buildbootDockerHostNetwork")) {
network.set("host")
}
docker {
if (project.hasProperty("buildbootDockerHostNetwork")) {
bindHostToBuilder.set(true)
}
verboseLogging.set(true)
publishRegistry {
username.set(providers.gradleProperty("mavenUser").getOrNull())
password.set(providers.gradleProperty("mavenPassword").getOrNull())
email.set(providers.gradleProperty("mavenEmail").getOrNull())
url.set("https://nexus.knecon.com:5001/")
}
}
}
@@ -1,84 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service</artifactId>
<version>0.1-SNAPSHOT</version>
</parent>
<artifactId>layoutparser-service-server</artifactId>
<dependencies>
<dependency>
<groupId>com.knecon.fforesight</groupId>
<artifactId>layoutparser-service-processor</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<!-- generate git.properties for exposure in /info -->
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>revision</goal>
</goals>
<configuration>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<gitDescribe>
<tags>true</tags>
</gitDescribe>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<!-- repackages the generated jar into a runnable fat-jar and makes it
executable -->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
<configuration>
<executable>true</executable>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,112 @@
[
{
"name": "ch.qos.logback.classic.AsyncAppender",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.classic.encoder.PatternLayoutEncoder",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.classic.pattern.DateConverter",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.classic.pattern.LevelConverter",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.classic.pattern.LineSeparatorConverter",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.classic.pattern.LoggerConverter",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.classic.pattern.MessageConverter",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.classic.pattern.ThreadConverter",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.core.ConsoleAppender",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
},
{
"name": "ch.qos.logback.core.FileAppender",
"allDeclaredConstructors": true,
"allPublicConstructors": true,
"allDeclaredMethods": true,
"allPublicMethods": true,
"allDeclaredFields": true,
"allPublicFields": true,
"allDeclaredClasses": true,
"allPublicClasses": true
}
]
@@ -6,8 +6,9 @@ import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
class ApplicationTests extends BaseTest {
@Test
void contextLoads() {
}
@Test
void contextLoads() {
}
}
@@ -22,11 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
@@ -45,8 +45,9 @@ public class BdrJsonBuildTest extends BaseTest {
protected Document buildGraph(File filename) {
try (InputStream inputStream = new FileInputStream(filename)) {
PDDocument pdDocument = Loader.loadPDF(inputStream);
return layoutParsingPipeline.parseLayoutWithTimer(LayoutParsingType.TAAS, pdDocument, new ImageServiceResponse(), new TableServiceResponse());
try (PDDocument pdDocument = Loader.loadPDF(inputStream)) {
return layoutParsingPipeline.parseLayoutWithTimer(LayoutParsingType.TAAS, pdDocument, new ImageServiceResponse(), new TableServiceResponse());
}
}
}
@@ -94,12 +95,13 @@ public class BdrJsonBuildTest extends BaseTest {
private static void visualizeSemanticNodes(File file, File resultingFileName, Document document, TextBlock textBlock) throws IOException {
try (var fileStream = new FileInputStream(file); var outputStream = new FileOutputStream(resultingFileName)) {
PDDocument pdDocument = Loader.loadPDF(fileStream);
try (var fileStream = new FileInputStream(file);//
PDDocument pdDocument = Loader.loadPDF(fileStream);//
var outputStream = new FileOutputStream(resultingFileName)//
) {
PdfDraw.drawDocumentGraph(pdDocument, document);
PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
pdDocument.save(outputStream);
pdDocument.close();
}
}
@@ -30,10 +30,10 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import lombok.AllArgsConstructor;
@@ -11,7 +11,7 @@ import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
class BoundaryTest {
@@ -13,8 +13,8 @@ import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
import lombok.SneakyThrows;
@@ -24,36 +24,34 @@ public class BuildDocumentGraphTest extends BaseTest {
@Autowired
protected LayoutParsingPipeline layoutParsingPipeline;
@Test
@Disabled
public void buildMetolachlor() {
Document documentGraph = buildGraph("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
assertEquals(221, documentGraph.getPages().size());
assertEquals(220 , documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
assertEquals(0 , documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
}
@SneakyThrows
protected Document buildGraph(String filename) {
if (!filename.endsWith(".pdf")) {
filename = filename + ".pdf";
}
if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
if (filename.equals("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf")) {
prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/S-Metolachlor_RAR_01_Volume_1_2018-09-06.IMAGE_INFO.json");
} else {
prepareStorage(filename);
}
ClassPathResource fileResource = new ClassPathResource(filename);
try (InputStream inputStream = fileResource.getInputStream()) {
PDDocument pdDocument = Loader.loadPDF(inputStream);
return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER, pdDocument, layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID), new TableServiceResponse());
try (InputStream inputStream = fileResource.getInputStream(); PDDocument pdDocument = Loader.loadPDF(inputStream)) {
return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
pdDocument,
layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
new TableServiceResponse());
}
}
}
@@ -11,7 +11,7 @@ import org.springframework.core.io.ClassPathResource;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import lombok.SneakyThrows;
@@ -15,10 +15,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import lombok.SneakyThrows;
@@ -46,7 +46,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentGraphTest {
@Disabled
public void writeJsonForFileTest() {
var resource = new ClassPathResource("files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
var resource = new ClassPathResource("files/1 Abamectin_prr.pdf");
writeJsons(resource.getFile().toPath());
}
@@ -11,11 +11,11 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentGraphMapper;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.PropertiesMapper;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentGraphMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@@ -26,7 +26,7 @@ public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
@SneakyThrows
public void testGraphMapping() {
String filename = "files/new/crafted document";
String filename = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf";
Document document = buildGraph(filename);
DocumentData documentData = DocumentDataMapper.toDocumentData(document);
@@ -10,8 +10,8 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
import lombok.SneakyThrows;
@@ -63,12 +63,12 @@ public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
File tmpFile = File.createTempFile(filename, "SEMANTIC_NODES_BBOX.pdf");
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
try (var fileStream = fileResource.getInputStream()) {
PDDocument pdDocument = Loader.loadPDF(fileStream);
PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
pdDocument.save(tmpFile);
pdDocument.close();
try (var fileStream = fileResource.getInputStream();//
PDDocument pdDocument = Loader.loadPDF(fileStream)//
) {
PdfDraw.drawDocumentGraph(pdDocument, documentGraph);
PdfDraw.drawTextBlock(pdDocument, textBlock, PdfDraw.Options.builder().stroke(true).strokeWidth(0.1f).strokeColor(Color.YELLOW).build());
pdDocument.save(tmpFile);
}
}

Some files were not shown because too many files have changed in this diff Show More