Compare commits

...
19 Commits
Author SHA1 Message Date
Kilian Schuettler 2a55654fcf add simplifiedText 2023-07-31 15:30:03 +02:00
Kilian Schüttler 7496914b37 Merge branch 'RED-6725' into 'main'
include openfeign for tenant-commons

See merge request fforesight/layout-parser!6
2023-07-31 15:02:33 +02:00
Kilian Schuettler c8ace585e1 include openfeign for tenant-commons 2023-07-31 14:40:57 +02:00
Kilian Schüttler 69c5f80c8c Merge branch 'RED-6725' into 'main'
Red 6725

See merge request fforesight/layout-parser!5
2023-07-31 13:19:39 +02:00
Kilian Schuettler 79d27189fd move Properties names to DocumentStructure 2023-07-31 12:54:49 +02:00
Kilian Schuettler 75bac72c05 fix dsljson 2023-07-29 02:44:22 +02:00
Kilian Schuettler c5e6271dc3 fix fileIds 2023-07-29 01:59:32 +02:00
Kilian Schüttler 5561dd5e95 Merge branch 'RED-6725' into 'main'
Red 6725

See merge request fforesight/layout-parser!4
2023-07-28 17:46:40 +02:00
Kilian Schuettler 041b633742 use correct repo 2023-07-28 17:42:18 +02:00
Kilian Schuettler 715426bd3b remove hardcoded version 2023-07-28 17:34:59 +02:00
Kilian Schuettler 464b8053fe configure maven-publish plugin 2023-07-28 17:26:05 +02:00
Kilian Schuettler 2fece83c7c remove root build.gradle 2023-07-28 16:15:53 +02:00
Kilian Schuettler ad03ef1922 move queue to server package, so i can easily import processor as a library for redaction-service 2023-07-28 16:15:31 +02:00
Christoph Schabert cc44100e4e add root build.gradle.kts 2023-07-28 13:46:06 +02:00
Christoph Schabert 5d1c1ae406 Update gradle.properties.kts 2023-07-27 18:00:31 +02:00
Kilian Schuettler f72838b0be removed unnecessary dependency 2023-07-27 17:15:22 +02:00
Kilian Schuettler 6388898cc0 added comments for native build investigation 2023-07-27 17:12:16 +02:00
Kilian Schuettler 72d1e6271a more refactoring, added a comment 2023-07-27 14:35:40 +02:00
Kilian Schuettler 299b5be385 package refactoring in processor 2023-07-27 14:28:09 +02:00
99 changed files with 723 additions and 596 deletions
@@ -1,24 +1,16 @@
plugins {
java
`java-library`
`maven-publish`
pmd
checkstyle
jacoco
}
group = "com.knecon.fforesight"
version = "0.1-SNAPSHOT"
java.sourceCompatibility = JavaVersion.VERSION_17
java.targetCompatibility = JavaVersion.VERSION_17
tasks.jacocoTestReport {
reports {
xml.required.set(false)
csv.required.set(false)
html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
}
}
tasks.pmdMain {
pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
}
@@ -29,6 +21,9 @@ tasks.pmdTest {
tasks.named<Test>("test") {
useJUnitPlatform()
reports {
junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
}
}
tasks.test {
@@ -40,14 +35,38 @@ tasks.jacocoTestReport {
reports {
xml.required.set(true)
csv.required.set(false)
html.outputLocation.set(layout.buildDirectory.dir("jacocoHtml"))
}
}
allprojects {
publishing {
publications {
create<MavenPublication>(name) {
from(components["java"])
}
}
repositories {
maven {
url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
credentials {
username = providers.gradleProperty("mavenUser").getOrNull();
password = providers.gradleProperty("mavenPassword").getOrNull();
}
}
}
}
}
java {
withJavadocJar()
}
repositories {
mavenLocal()
mavenCentral()
maven {
url = uri("https://nexus.knecon.com/repository/gindev/");
url = uri("https://nexus.knecon.com/repository/gindev/")
credentials {
username = providers.gradleProperty("mavenUser").getOrNull();
password = providers.gradleProperty("mavenPassword").getOrNull();
+1 -1
View File
@@ -1 +1 @@
version = 0.1.0
version = 0.1-SNAPSHOT
@@ -1,5 +1,7 @@
package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
import java.awt.geom.Rectangle2D;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
@@ -21,6 +23,39 @@ public class DocumentStructure {
EntryData root;
public static class TableProperties {
public static final String NUMBER_OF_ROWS = "numberOfRows";
public static final String NUMBER_OF_COLS = "numberOfCols";
}
public static class ImageProperties {
public static final String TRANSPARENT = "transparent";
public static final String IMAGE_TYPE = "imageType";
public static final String POSITION = "position";
}
public static class TableCellProperties {
public static final String B_BOX = "bBox";
public static final String ROW = "row";
public static final String COL = "col";
public static final String HEADER = "header";
}
public static final String RECTANGLE_DELIMITER = ";";
public static Rectangle2D parseRectangle2D(String bBox) {
List<Float> floats = Arrays.stream(bBox.split(RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
}
public EntryData get(List<Integer> tocId) {
if (tocId.isEmpty()) {
@@ -66,6 +101,7 @@ public class DocumentStructure {
Map<String, String> properties;
List<EntryData> children;
@Override
public String toString() {
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.model.text;
package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
import lombok.AllArgsConstructor;
import lombok.Builder;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.model.text;
package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
import java.util.ArrayList;
import java.util.List;
@@ -18,6 +18,7 @@ public record LayoutParsingRequest(
String textBlockFileStorageId,
String positionBlockFileStorageId,
String pageFileStorageId,
String simplifiedTextStorageId,
String sectionGridStorageId) {
}
@@ -3,10 +3,15 @@ plugins {
id("io.freefair.lombok") version "8.1.0"
}
description = "layoutparser-service-processor"
dependencies {
implementation(project(":layoutparser-service-internal-api"))
implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0")
implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.36.0") {
exclude("org.springframework.boot", "spring-boot-starter-security")
exclude("org.springframework.boot", "spring-boot-starter-validation")
}
implementation("com.knecon.fforesight:tenant-commons:0.10.0")
implementation("com.iqser.red.commons:storage-commons:2.1.0")
@@ -15,7 +20,4 @@ dependencies {
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.15.0-rc2")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0-rc2")
implementation("org.springframework.boot:spring-boot-starter-web:3.0.6")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
}
description = "layoutparser-service-processor"
@@ -10,14 +10,16 @@ import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.factory.DocumentGraphFactory;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
import com.knecon.fforesight.service.layoutparser.processor.services.PdfParsingService;
import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
@@ -43,6 +45,7 @@ public class LayoutParsingPipeline {
private final TaasClassificationService taasClassificationService;
private final RedactManagerClassificationService redactManagerClassificationService;
private final DocuMineClassificationService docuMineClassificationService;
private final SimplifiedSectionTextService simplifiedSectionTextService;
public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -52,12 +55,12 @@ public class LayoutParsingPipeline {
try (PDDocument originDocument = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId())) {
ImageServiceResponse imageServiceResponse = new ImageServiceResponse();
if (layoutParsingRequest.imagesFileStorageId().isPresent()) {
imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.pageFileStorageId());
imageServiceResponse = layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId().get());
}
TableServiceResponse tableServiceResponse = new TableServiceResponse();
if (layoutParsingRequest.tablesFileStorageId().isPresent()) {
tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.pageFileStorageId());
tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
}
Document documentGraph = parseLayout(layoutParsingRequest.layoutParsingType(), originDocument, imageServiceResponse, tableServiceResponse);
@@ -65,7 +68,7 @@ public class LayoutParsingPipeline {
layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
if (layoutParsingRequest.layoutParsingType().equals(LayoutParsingType.TAAS)) {
var researchDocumentData = TaasDocumentDataMapper.fromDocument(documentGraph);
layoutParsingStorageService.storeResearchDocumentData(layoutParsingRequest, researchDocumentData);
@@ -15,15 +15,12 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
@@ -90,26 +87,6 @@ public class LayoutParsingStorageService {
}
public DocumentData readDocumentData(LayoutParsingRequest layoutParsingRequest) throws IOException {
DocumentPage[] documentPageData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.pageFileStorageId(), DocumentPage[].class);
DocumentTextData[] documentTextDataBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
layoutParsingRequest.textBlockFileStorageId(),
DocumentTextData[].class);
DocumentPositionData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
layoutParsingRequest.positionBlockFileStorageId(),
DocumentPositionData[].class);
DocumentStructure tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), layoutParsingRequest.structureFileStorageId(), DocumentStructure.class);
return DocumentData.builder()
.documentStructure(tableOfContentsData)
.documentPositions(atomicPositionBlockData)
.documentTextData(documentTextDataBlockData)
.documentPages(documentPageData)
.build();
}
private File createTempFile(String filenamePrefix, String filenameSuffix) throws IOException {
File tempFile = Files.createTempFile(filenamePrefix, filenameSuffix).toFile();
@@ -134,4 +111,10 @@ public class LayoutParsingStorageService {
}
}
public void storeSimplifiedText(LayoutParsingRequest layoutParsingRequest, SimplifiedText simplifiedText) {
storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.simplifiedTextStorageId(), simplifiedText);
}
}
@@ -1,14 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
@Data
public class Classification {
private Map<String, Float> probabilities = new HashMap<>();
private String label;
}
@@ -1,11 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class FilterGeometry {
private ImageSize imageSize;
private ImageFormat imageFormat;
}
@@ -1,12 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class Filters {
private FilterGeometry geometry;
private Probability probability;
private boolean allPassed;
}
@@ -1,11 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class Geometry {
private float width;
private float height;
}
@@ -1,12 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class ImageFormat {
private float quotient;
private boolean tooTall;
private boolean tooWide;
}
@@ -1,14 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class ImageMetadata {
private Classification classification;
private Position position;
private Geometry geometry;
private Filters filters;
private boolean alpha;
}
@@ -1,12 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class ImageSize {
private float quotient;
private boolean tooLarge;
private boolean tooSmall;
}
@@ -1,14 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class Position {
private float x1;
private float x2;
private float y1;
private float y2;
private int pageNumber;
}
@@ -1,10 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
import lombok.Data;
@Data
public class Probability {
private boolean unconfident;
}
@@ -1,13 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
import lombok.Data;
@Data
public class PageInfo {
private int number;
private int rotation;
private float width;
private float height;
}
@@ -1,14 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class TableData {
private PageInfo pageInfo;
private List<TableCells> tableCells = new ArrayList<>();
}
@@ -1,5 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
public interface GenericSemanticNode extends SemanticNode {
}
@@ -1,112 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
import java.awt.geom.Rectangle2D;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
public class PropertiesMapper {
public static final String TRANSPARENT = "transparent";
public static final String IMAGE_TYPE = "imageType";
public static final String POSITION = "position";
public static final String ROW = "row";
public static final String COL = "col";
public static final String HEADER = "header";
public static final String B_BOX = "bBox";
public static final String NUMBER_OF_ROWS = "numberOfRows";
public static final String NUMBER_OF_COLS = "numberOfCols";
public static Map<String, String> buildImageProperties(Image image) {
Map<String, String> properties = new HashMap<>();
properties.put(IMAGE_TYPE, image.getImageType().toString());
properties.put(TRANSPARENT, String.valueOf(image.isTransparent()));
properties.put(POSITION, RectangleTransformations.toString(image.getPosition()));
return properties;
}
public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
Map<String, String> properties = new HashMap<>();
properties.put(ROW, String.valueOf(tableCell.getRow()));
properties.put(COL, String.valueOf(tableCell.getCol()));
properties.put(HEADER, String.valueOf(tableCell.isHeader()));
if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
throw new IllegalArgumentException("TableCell can only occur on a single page!");
}
String bBoxString = RectangleTransformations.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
properties.put(B_BOX, bBoxString);
return properties;
}
public static Map<String, String> buildTableProperties(Table table) {
Map<String, String> properties = new HashMap<>();
properties.put(NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
properties.put(NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
return properties;
}
public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
builder.imageType(parseImageType(properties.get(IMAGE_TYPE)));
builder.transparent(Boolean.parseBoolean(properties.get(TRANSPARENT)));
builder.position(parseRectangle2D(properties.get(POSITION)));
}
public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
builder.row(Integer.parseInt(properties.get(ROW)));
builder.col(Integer.parseInt(properties.get(COL)));
builder.header(Boolean.parseBoolean(properties.get(HEADER)));
builder.bBox(parseRectangle2D(properties.get(B_BOX)));
}
public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
builder.numberOfRows(Integer.parseInt(properties.get(NUMBER_OF_ROWS)));
builder.numberOfCols(Integer.parseInt(properties.get(NUMBER_OF_COLS)));
}
private static ImageType parseImageType(String imageType) {
return switch (imageType) {
case "LOGO" -> ImageType.LOGO;
case "FORMULA" -> ImageType.FORMULA;
case "SIGNATURE" -> ImageType.SIGNATURE;
case "OCR" -> ImageType.OCR;
default -> ImageType.OTHER;
};
}
public static String toString(Rectangle2D rectangle2D) {
return String.format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
}
public static Rectangle2D parseRectangle2D(String bBox) {
List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
}
}
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph;
package com.knecon.fforesight.service.layoutparser.processor.model.graph;
import static java.lang.String.format;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph;
package com.knecon.fforesight.service.layoutparser.processor.model.graph;
import static java.lang.String.format;
@@ -8,13 +8,13 @@ import java.util.List;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
public enum EntityType {
ENTITY,
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
import java.awt.geom.Rectangle2D;
import java.util.Collection;
@@ -11,9 +11,9 @@ import java.util.Map;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
import lombok.AccessLevel;
@@ -1,9 +1,9 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.entity;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.entity;
import java.awt.geom.Rectangle2D;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
@@ -12,10 +12,10 @@ import java.util.stream.Stream;
import com.amazonaws.services.kms.model.NotFoundException;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -0,0 +1,5 @@
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
public interface GenericSemanticNode extends SemanticNode {
}
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
@@ -9,10 +9,10 @@ import java.util.Map;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.Locale;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.LinkedList;
@@ -6,9 +6,9 @@ import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,13 +1,13 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,14 +1,14 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import static java.lang.String.format;
@@ -13,12 +13,12 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.EntityType;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.EntityType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
public interface SemanticNode {
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import static java.lang.String.format;
@@ -11,10 +11,10 @@ import java.util.stream.IntStream;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.nodes;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
@@ -8,10 +8,10 @@ import java.util.Map;
import java.util.Set;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import static java.lang.String.format;
@@ -13,9 +13,9 @@ import java.util.Map;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
import lombok.AccessLevel;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import static java.lang.String.format;
@@ -11,8 +11,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import lombok.AccessLevel;
import lombok.Data;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import static java.lang.String.format;
@@ -10,8 +10,8 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
public interface TextBlock extends CharSequence {
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.graph.textblock;
package com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock;
import java.util.Set;
import java.util.function.BiConsumer;
@@ -2,7 +2,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model.image;
import java.awt.geom.Rectangle2D;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
import lombok.Data;
import lombok.NonNull;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter;
package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
import java.util.ArrayList;
import java.util.Collection;
@@ -8,8 +8,8 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter;
package com.knecon.fforesight.service.layoutparser.processor.python_api.adapter;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
@@ -9,10 +9,10 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
import lombok.RequiredArgsConstructor;
@@ -0,0 +1,20 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import java.util.HashMap;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Classification {
private Map<String, Float> probabilities = new HashMap<>();
private String label;
}
@@ -0,0 +1,17 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FilterGeometry {
private ImageSize imageSize;
private ImageFormat imageFormat;
}
@@ -0,0 +1,18 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Filters {
private FilterGeometry geometry;
private Probability probability;
private boolean allPassed;
}
@@ -0,0 +1,17 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Geometry {
private float width;
private float height;
}
@@ -0,0 +1,18 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ImageFormat {
private float quotient;
private boolean tooTall;
private boolean tooWide;
}
@@ -0,0 +1,20 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ImageMetadata {
private Classification classification;
private Position position;
private Geometry geometry;
private Filters filters;
private boolean alpha;
}
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.image;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import java.util.ArrayList;
import java.util.List;
@@ -6,9 +6,15 @@ import java.util.List;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ImageServiceResponse {
private String dossierId;
@@ -0,0 +1,18 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ImageSize {
private float quotient;
private boolean tooLarge;
private boolean tooSmall;
}
@@ -0,0 +1,20 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Position {
private float x1;
private float x2;
private float y1;
private float y2;
private int pageNumber;
}
@@ -0,0 +1,16 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.image;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Probability {
private boolean unconfident;
}
@@ -0,0 +1,19 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class PageInfo {
private int number;
private int rotation;
private float width;
private float height;
}
@@ -1,14 +1,14 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@RequiredArgsConstructor
public class PdfTableCell {
private float x0;
@@ -1,10 +1,14 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class TableCells {
private float x0;
@@ -0,0 +1,20 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class TableData {
private PageInfo pageInfo;
private List<TableCells> tableCells = new ArrayList<>();
}
@@ -1,11 +1,17 @@
package com.knecon.fforesight.service.layoutparser.processor.adapter.model.table;
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class TableServiceResponse {
private String dossierId;
@@ -1,40 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.queue;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import lombok.RequiredArgsConstructor;
@Configuration
@RequiredArgsConstructor
public class MessagingConfiguration {
@Bean
public Queue layoutparsingRequestQueue() {
return QueueBuilder.durable(LAYOUT_PARSING_REQUEST_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingResponseQueue() {
return QueueBuilder.durable(LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingDLQ() {
return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
}
}
@@ -10,8 +10,8 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -19,7 +19,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.image.Classifi
import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
import com.knecon.fforesight.service.layoutparser.processor.services.blockification.TaasBlockificationService;
@@ -12,7 +12,7 @@ import java.util.Map;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
@@ -20,11 +20,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import lombok.RequiredArgsConstructor;
@@ -0,0 +1,26 @@
package com.knecon.fforesight.service.layoutparser.processor.services;
import java.util.List;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
@Service
public class SimplifiedSectionTextService {
public SimplifiedText toSimplifiedText(Document document) {
List<SimplifiedSectionText> simplifiedSectionTexts = document.getMainSections().stream().map(this::toSimplifiedSectionText).toList();
return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build();
}
private SimplifiedSectionText toSimplifiedSectionText(Section section) {
return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
}
}
@@ -15,7 +15,7 @@ import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.services.parsing.PDFLinesTextStripper;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy;
@@ -20,17 +20,17 @@ import com.knecon.fforesight.service.layoutparser.processor.model.Classification
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.utils.IdBuilder;
import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
@@ -1,10 +1,10 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import lombok.AccessLevel;
import lombok.Builder;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import java.awt.geom.AffineTransform;
import java.awt.geom.Rectangle2D;
@@ -11,7 +11,7 @@ import java.util.Objects;
import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
@@ -14,9 +14,9 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.utils.TableMergingUtility;
import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import static java.util.Collections.emptyList;
@@ -11,12 +11,12 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
import lombok.experimental.UtilityClass;
@@ -1,11 +1,11 @@
package com.knecon.fforesight.service.layoutparser.processor.factory;
package com.knecon.fforesight.service.layoutparser.processor.services.factory;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import lombok.AccessLevel;
import lombok.experimental.FieldDefaults;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
@@ -12,14 +12,14 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.experimental.UtilityClass;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.redaction;
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.util.Arrays;
import java.util.HashSet;
@@ -12,21 +12,21 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Footer;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Header;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Headline;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;
import lombok.experimental.UtilityClass;
@@ -0,0 +1,99 @@
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
import java.util.Map;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
public class PropertiesMapper {
public static Map<String, String> buildImageProperties(Image image) {
Map<String, String> properties = new HashMap<>();
properties.put(DocumentStructure.ImageProperties.IMAGE_TYPE, image.getImageType().toString());
properties.put(DocumentStructure.ImageProperties.TRANSPARENT, String.valueOf(image.isTransparent()));
properties.put(DocumentStructure.ImageProperties.POSITION, toString(image.getPosition()));
return properties;
}
public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
Map<String, String> properties = new HashMap<>();
properties.put(DocumentStructure.TableCellProperties.ROW, String.valueOf(tableCell.getRow()));
properties.put(DocumentStructure.TableCellProperties.COL, String.valueOf(tableCell.getCol()));
properties.put(DocumentStructure.TableCellProperties.HEADER, String.valueOf(tableCell.isHeader()));
if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
throw new IllegalArgumentException("TableCell can only occur on a single page!");
}
String bBoxString = toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
properties.put(DocumentStructure.TableCellProperties.B_BOX, bBoxString);
return properties;
}
public static Map<String, String> buildTableProperties(Table table) {
Map<String, String> properties = new HashMap<>();
properties.put(DocumentStructure.TableProperties.NUMBER_OF_ROWS, String.valueOf(table.getNumberOfRows()));
properties.put(DocumentStructure.TableProperties.NUMBER_OF_COLS, String.valueOf(table.getNumberOfCols()));
return properties;
}
public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
builder.imageType(parseImageType(properties.get(DocumentStructure.ImageProperties.IMAGE_TYPE)));
builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructure.ImageProperties.TRANSPARENT)));
builder.position(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.ImageProperties.POSITION)));
}
public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
builder.row(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.ROW)));
builder.col(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.COL)));
builder.header(Boolean.parseBoolean(properties.get(DocumentStructure.TableCellProperties.HEADER)));
builder.bBox(DocumentStructure.parseRectangle2D(properties.get(DocumentStructure.TableCellProperties.B_BOX)));
}
public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_ROWS)));
builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_COLS)));
}
private static ImageType parseImageType(String imageType) {
return switch (imageType) {
case "LOGO" -> ImageType.LOGO;
case "FORMULA" -> ImageType.FORMULA;
case "SIGNATURE" -> ImageType.SIGNATURE;
case "OCR" -> ImageType.OCR;
default -> ImageType.OTHER;
};
}
public static String toString(Rectangle2D rectangle2D) {
return String.format("%f%s%f%s%f%s%f",
rectangle2D.getX(),
DocumentStructure.RECTANGLE_DELIMITER,
rectangle2D.getY(),
DocumentStructure.RECTANGLE_DELIMITER,
rectangle2D.getWidth(),
DocumentStructure.RECTANGLE_DELIMITER,
rectangle2D.getHeight());
}
}
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.mapper.taas;
package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
import java.awt.geom.Rectangle2D;
import java.util.List;
@@ -13,13 +13,13 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.Researc
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.RowData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.StructureObject;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.TableData;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
public class TaasDocumentDataMapper {
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.knecon.fforesight.service.layoutparser.processor.parsing;
package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
import java.io.IOException;
import java.io.InputStream;
@@ -1,4 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.parsing;
package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
import java.awt.color.CMMException;
import java.awt.geom.Point2D;
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.knecon.fforesight.service.layoutparser.processor.parsing;
package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
import java.io.BufferedInputStream;
import java.io.IOException;
@@ -9,7 +9,7 @@ import java.util.stream.Collectors;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import lombok.experimental.UtilityClass;
@@ -15,11 +15,11 @@ import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import lombok.AccessLevel;
import lombok.Builder;
@@ -18,7 +18,7 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
@@ -1,23 +1,27 @@
import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
application
id("com.knecon.fforesight.java-conventions")
id("org.springframework.boot") version "3.1.2"
id("io.spring.dependency-management") version "1.1.0"
id("org.sonarqube") version "4.2.1.3168"
id("io.freefair.lombok") version "8.1.0"
//id("org.graalvm.buildtools.native") version "0.9.23"
// id("org.graalvm.buildtools.native") version "0.9.23"
}
description = "layoutparser-service-server"
dependencies {
implementation(project(":layoutparser-service-processor"))
implementation(project(":layoutparser-service-internal-api"))
implementation("com.iqser.red.commons:storage-commons:2.1.0")
implementation("com.knecon.fforesight:tenant-commons:0.10.0")
implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.2")
implementation("com.amazonaws:aws-java-sdk-s3:1.12.514")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.6")
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
// for integration testing only
testImplementation(project(":layoutparser-service-internal-api"))
@@ -30,17 +34,17 @@ dependencies {
testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.6")
testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.19.0")
}
description = "layoutparser-service-server"
java {
withJavadocJar()
}
/*
// AOT seems to be the name of the generated classes for native images
// They are added as a SourceSet, and therefore checkstyle and pmd try to run on it
//TODO: implement native build using GRAALVM, currently the problem is, logback needs runtime configuration using reflections or something
//kinda weird behavior, since the buildArgs below seem to have no effect at all
//https://stackoverflow.com/questions/72770461/graalvm-native-image-can-not-compile-logback-dependencies
We traced the issue to storage-commons and there to azure blob storage, this is today an open bug with azure spring
https://github.com/Azure/azure-sdk-for-java/issues/36001
//AOT is springs ahead of time compilation, it is a code analysis which generates new code, therefore checkstyle and pmd can be disabled on them.
tasks.named<Checkstyle>("checkstyleAot") {
enabled = false
}
@@ -53,28 +57,15 @@ tasks.named<Pmd>("pmdAot") {
tasks.named<Pmd>("pmdAotTest") {
enabled = false
}
*/
graalvmNative {
binaries {
all {
resources.autodetect()
buildArgs.add("--verbose")
buildArgs.add("--add-opens=java.base/java.nio=ALL-UNNAMED")
buildArgs.add("--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED")
buildArgs.add("--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED")
buildArgs.add("--trace-class-initialization=ch.qos.logback.classic.Logger")
buildArgs.add("--trace-object-instantiation=ch.qos.logback.core.AsyncAppenderBase\$Worker")
buildArgs.add("--initialize-at-build-time=org.slf4j.LoggerFactory,ch.qos.logback")
buildArgs.add("--initialize-at-run-time=io.netty")
}
sonarqube {
properties {
providers.gradleProperty("sonarToken").getOrNull()?.let { property("sonar.login", it) }
property("sonar.host.url", "https://sonarqube.knecon.com")
}
}
nativeBuild {
buildArgs("-H:ReflectionConfigurationFiles=../../../src/main/resources/reflection-config.json")
}
*/
tasks.named<BootBuildImage>("bootBuildImage") {
imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
if (project.hasProperty("buildbootDockerHostNetwork")) {
@@ -10,7 +10,7 @@ import org.springframework.context.annotation.Import;
import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.queue.MessagingConfiguration;
import com.knecon.fforesight.service.layoutparser.server.queue.MessagingConfiguration;
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
@@ -1,7 +1,4 @@
package com.knecon.fforesight.service.layoutparser.processor.queue;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE;
package com.knecon.fforesight.service.layoutparser.server.queue;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.Message;
@@ -12,6 +9,7 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -30,7 +28,7 @@ public class MessageHandler {
@RabbitHandler
@RabbitListener(queues = LAYOUT_PARSING_REQUEST_QUEUE)
@RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
@SneakyThrows
public void receiveLayoutParsingRequest(Message message) {
@@ -49,7 +47,7 @@ public class MessageHandler {
public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
rabbitTemplate.convertAndSend(LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
}
}
@@ -0,0 +1,38 @@
package com.knecon.fforesight.service.layoutparser.server.queue;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
import lombok.RequiredArgsConstructor;
@Configuration
@RequiredArgsConstructor
public class MessagingConfiguration {
@Bean
public Queue layoutparsingRequestQueue() {
return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingResponseQueue() {
return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
.withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
}
@Bean
public Queue layoutparsingDLQ() {
return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
}
}
@@ -2,6 +2,7 @@ info:
description: Layout Parser Service Processor
tenant-user-management-service.url: "http://tenant-user-management-service:8080/internal"
fforesight.tenants.remote: true
server:
@@ -22,11 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.mapper.taas.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
@@ -30,10 +30,10 @@ import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.SemanticNode;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import lombok.AllArgsConstructor;
@@ -11,7 +11,7 @@ import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import com.knecon.fforesight.service.layoutparser.processor.graph.Boundary;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.Boundary;
class BoundaryTest {
@@ -13,8 +13,8 @@ import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.server.utils.BaseTest;
import lombok.SneakyThrows;
@@ -11,7 +11,7 @@ import org.springframework.core.io.ClassPathResource;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import lombok.SneakyThrows;
@@ -15,10 +15,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import lombok.SneakyThrows;
@@ -11,11 +11,11 @@ import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.Do
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.DocumentGraphMapper;
import com.knecon.fforesight.service.layoutparser.processor.mapper.redaction.PropertiesMapper;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentGraphMapper;
import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows;
@@ -10,8 +10,8 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
import lombok.SneakyThrows;
@@ -21,11 +21,11 @@ import org.springframework.core.io.ClassPathResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.adapter.CvTableParsingAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.adapter.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.CvTableParsingAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.adapter.ImageServiceResponseAdapter;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -19,11 +19,11 @@ import org.apache.pdfbox.util.Matrix;
import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
import com.knecon.fforesight.service.layoutparser.processor.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.processor.utils.PdfVisualisationUtility;
import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;