Marmelator/layout-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 289 Wiki Activity

RED-8666

Browse Source
This commit is contained in:
maverickstuder 2024-03-04 09:23:30 +01:00
parent aef1146e8f
commit 2567d89fbb
6 changed files with 78 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java
View File

@ -6,5 +6,5 @@ public enum LayoutParsingType {
DOCUMINE, DOCUMINE,
DOCSTRUM, DOCSTRUM,
DOCSTRUM_XY DOCSTRUM_ROW_WISE
} }

12
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java
View File

@ -1,7 +1,7 @@
package com.knecon.fforesight.service.layoutparser.processor; package com.knecon.fforesight.service.layoutparser.processor;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType.DOCSTRUM; import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType.DOCSTRUM;
import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType.DOCSTRUM_XY; import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType.DOCSTRUM_ROW_WISE;
import static java.lang.String.format; import static java.lang.String.format;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
@ -266,8 +266,8 @@ public class LayoutParsingPipeline {
case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
case DOCSTRUM -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), false); case DOCSTRUM -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), true);
case DOCSTRUM_XY -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), true); case DOCSTRUM_ROW_WISE -> docstrumBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), false);
}; };
classificationPage.setCleanRulings(cleanRulings); classificationPage.setCleanRulings(cleanRulings);
classificationPage.setRotation(rotation); classificationPage.setRotation(rotation);
@ -291,7 +291,7 @@ public class LayoutParsingPipeline {
tableExtractionService.extractTables(cleanRulings, classificationPage); tableExtractionService.extractTables(cleanRulings, classificationPage);
if (layoutParsingType == DOCSTRUM || layoutParsingType == DOCSTRUM_XY) { if (layoutParsingType == DOCSTRUM || layoutParsingType == DOCSTRUM_ROW_WISE) {
// docstrumBlockificationService.combineBlocks(classificationPage); //todo 8666 // docstrumBlockificationService.combineBlocks(classificationPage); //todo 8666
} }
@ -310,12 +310,12 @@ public class LayoutParsingPipeline {
case TAAS -> taasClassificationService.classifyDocument(classificationDocument); case TAAS -> taasClassificationService.classifyDocument(classificationDocument);
case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument); case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument); case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument);
case DOCSTRUM_XY -> redactManagerClassificationService.classifyDocument(classificationDocument); case DOCSTRUM_ROW_WISE -> redactManagerClassificationService.classifyDocument(classificationDocument);
} }
log.info("Building Sections for {}", identifier); log.info("Building Sections for {}", identifier);
if (layoutParsingType == DOCSTRUM || layoutParsingType == DOCSTRUM_XY) { if (layoutParsingType == DOCSTRUM || layoutParsingType == DOCSTRUM_ROW_WISE) {
// Currently for debugging return paragraphs as sections, because there is a merging logic in sectionBuilder // Currently for debugging return paragraphs as sections, because there is a merging logic in sectionBuilder
List<ClassificationSection> sections = new ArrayList<>(); List<ClassificationSection> sections = new ArrayList<>();
for (var page : classificationPages) { for (var page : classificationPages) {

4
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java
View File

@ -35,10 +35,10 @@ public class DocstrumBlockificationService {
static final float THRESHOLD = 2f; static final float THRESHOLD = 2f;
public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, boolean xyOder) { public ClassificationPage blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, boolean columnWise) {
List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>(); List<AbstractPageBlock> abstractPageBlocks = new ArrayList<>();
var zones = docstrumSegmentationService.segmentPage(textPositions, xyOder); var zones = docstrumSegmentationService.segmentPage(textPositions, columnWise);
zones.forEach(zone -> { zones.forEach(zone -> {
List<TextPositionSequence> textPositionSequences = new ArrayList<>(); List<TextPositionSequence> textPositionSequences = new ArrayList<>();

162
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/UnsupervisedReadingOrderDetector.java
View File

@ -4,8 +4,12 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone; import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
import lombok.Getter; import lombok.Getter;
@ -47,17 +51,17 @@ public class UnsupervisedReadingOrderDetector {
switch (spatialReasoningRule) { switch (spatialReasoningRule) {
case COLUMN_WISE: case COLUMN_WISE:
if (useRenderingOrder) { if (useRenderingOrder) {
zoneComparator = (Zone z1, Zone z2, double t) -> getBeforeInReadingVertical(z1, z2, t) || getBeforeInRendering(z1, z2); zoneComparator = (Zone z1, Zone z2, double t) -> getBeforeInReadingColumnWise(z1, z2, t) || getBeforeInRendering(z1, z2);
} else { } else {
zoneComparator = this::getBeforeInReadingVertical; zoneComparator = this::getBeforeInReadingColumnWise;
} }
break; break;
case ROW_WISE: case ROW_WISE:
if (useRenderingOrder) { if (useRenderingOrder) {
zoneComparator = (Zone z1, Zone z2, double t) -> getBeforeInReadingHorizontal(z1, z2, t) || getBeforeInRendering(z1, z2); zoneComparator = (Zone z1, Zone z2, double t) -> getBeforeInReadingRowWise(z1, z2, t) || getBeforeInRendering(z1, z2);
} else { } else {
zoneComparator = this::getBeforeInReadingHorizontal; zoneComparator = this::getBeforeInReadingRowWise;
} }
break; break;
@ -159,8 +163,8 @@ public class UnsupervisedReadingOrderDetector {
private boolean getBeforeInReading(Zone z1, Zone z2, double tolerance) { private boolean getBeforeInReading(Zone z1, Zone z2, double tolerance) {
IntervalRelations xRelation = getIntervalRelationX(z1, z2, tolerance); IntervalRelations xRelation = getIntervalRelationX(z1, z2, tolerance).get(0);
IntervalRelations yRelation = getIntervalRelationY(z1, z2, tolerance); IntervalRelations yRelation = getIntervalRelationY(z1, z2, tolerance).get(0);
return xRelation == IntervalRelations.PRECEDES return xRelation == IntervalRelations.PRECEDES
|| yRelation == IntervalRelations.PRECEDES || yRelation == IntervalRelations.PRECEDES
@ -171,119 +175,87 @@ public class UnsupervisedReadingOrderDetector {
} }
private boolean getBeforeInReadingVertical(Zone z1, Zone z2, double tolerance) { private boolean getBeforeInReadingColumnWise(Zone z1, Zone z2, double tolerance) {
IntervalRelations xRelation = getIntervalRelationX(z1, z2, tolerance); IntervalRelations xRelation = getIntervalRelationX(z1, z2, tolerance).get(0);
IntervalRelations yRelation = getIntervalRelationY(z1, z2, tolerance); IntervalRelations yRelation = getIntervalRelationY(z1, z2, tolerance).get(0);
return getIntervalRelations(xRelation, yRelation); return getIntervalRelations(xRelation, yRelation);
} }
private static boolean getIntervalRelations(IntervalRelations relation1, IntervalRelations relation2) { private static boolean getIntervalRelations(IntervalRelations relation1, IntervalRelations relation2) {
return relation1 == IntervalRelations.PRECEDES return relation1 == IntervalRelations.PRECEDES //
|| relation1 == IntervalRelations.MEETS || relation1 == IntervalRelations.MEETS //
|| (relation2 == IntervalRelations.PRECEDES || relation2 == IntervalRelations.MEETS || relation2 == IntervalRelations.OVERLAPS) && // || relation1 == IntervalRelations.OVERLAPS && //
(relation1 == IntervalRelations.OVERLAPS (relation2 == IntervalRelations.PRECEDES //
|| relation1 == IntervalRelations.STARTS || relation2 == IntervalRelations.MEETS //
|| relation1 == IntervalRelations.FINISHES_INVERSE || relation2 == IntervalRelations.OVERLAPS) //
|| relation1 == IntervalRelations.EQUALS || ((relation2 == IntervalRelations.PRECEDES || relation2 == IntervalRelations.MEETS || relation2 == IntervalRelations.OVERLAPS) && //
|| relation1 == IntervalRelations.DURING (relation1 == IntervalRelations.STARTS //
|| relation1 == IntervalRelations.DURING_INVERSE || relation1 == IntervalRelations.FINISHES_INVERSE //
|| relation1 == IntervalRelations.FINISHES || relation1 == IntervalRelations.EQUALS //
|| relation1 == IntervalRelations.STARTS_INVERSE || relation1 == IntervalRelations.DURING //
|| relation1 == IntervalRelations.OVERLAPS_INVERSE); || relation1 == IntervalRelations.DURING_INVERSE //
|| relation1 == IntervalRelations.FINISHES //
|| relation1 == IntervalRelations.STARTS_INVERSE //
|| relation1 == IntervalRelations.OVERLAPS_INVERSE));
} }
private boolean getBeforeInReadingHorizontal(Zone z1, Zone z2, double tolerance) {
IntervalRelations xRelation = getIntervalRelationX(z1, z2, tolerance); private boolean getBeforeInReadingRowWise(Zone z1, Zone z2, double tolerance) {
IntervalRelations yRelation = getIntervalRelationY(z1, z2, tolerance);
return getIntervalRelations(yRelation, xRelation); IntervalRelations xRelations = getIntervalRelationX(z1, z2, tolerance).get(0);
IntervalRelations yRelations = getIntervalRelationY(z1, z2, tolerance).get(0);
return getIntervalRelations(yRelations, xRelations);
} }
private static IntervalRelations getIntervalRelationX(Zone z1, Zone z2, double t) { private static List<IntervalRelations> getIntervalRelationX(Zone z1, Zone z2, double t) {
double z1_minX = z1.getX(); return getIntervalRelation(new ImmutablePair<>(z1.getX(), z1.getX() + z1.getWidth()), new ImmutablePair<>(z2.getX(), z2.getX() + z2.getWidth()), t);
double z1_maxX = z1_minX + z1.getWidth(); }
double z2_minX = z2.getX();
double z2_maxX = z2_minX + z2.getWidth();
// this is very wrong: check https://www.cs.rug.nl/~aiellom/publications/ijdarNoi.pdf
if (z1_maxX < z2_minX - t) { private static List<IntervalRelations> getIntervalRelationY(Zone z1, Zone z2, double t) {
return IntervalRelations.PRECEDES;
} else if (z1_maxX >= z2_minX - t) { return getIntervalRelation(new ImmutablePair<>(z1.getY(), z1.getY() + z1.getHeight()), new ImmutablePair<>(z2.getY(), z2.getY() + z2.getHeight()), t);
return IntervalRelations.PRECEDES_INVERSE;
} else if (z2_minX - t <= z1_maxX && z1_maxX <= z2_minX + t) { }
return IntervalRelations.MEETS;
} else if (z2_minX - t > z1_maxX && z1_maxX > z2_minX + t) {
return IntervalRelations.MEETS_INVERSE; private static List<IntervalRelations> getIntervalRelation(Pair<Double, Double> a, Pair<Double, Double> b, double t) {
} else if (z1_minX < z2_minX - t && (z2_minX + t < z1_maxX && z1_maxX < z2_maxX - t)) {
return IntervalRelations.OVERLAPS; var intervalRelations = getIntervalRelation(a, b, t, false);
} else if (z1_minX >= z2_minX - t && (z2_minX + t >= z1_maxX && z1_maxX >= z2_maxX - t)) { intervalRelations.addAll(getIntervalRelation(b, a, t, true));
return IntervalRelations.OVERLAPS_INVERSE; if ((b.getLeft() - t <= a.getLeft() && a.getLeft() <= b.getLeft() + t) && (b.getRight() - t <= a.getRight() && a.getRight() <= b.getRight() + t)) {
} else if (z2_minX - t <= z1_minX && z1_minX <= z2_minX + t && z1_maxX < z2_maxX - t) { intervalRelations.add(IntervalRelations.EQUALS);
return IntervalRelations.STARTS;
} else if (z2_minX - t > z1_minX && z1_minX > z2_minX + t && z1_maxX >= z2_maxX - t) {
return IntervalRelations.STARTS_INVERSE;
} else if (z1_minX > z2_minX + t && z1_maxX < z2_maxX - t) {
return IntervalRelations.DURING;
} else if (z1_minX <= z2_minX + t && z1_maxX >= z2_maxX - t) {
return IntervalRelations.DURING_INVERSE;
} else if (z1_minX > z2_minX + t && (z2_maxX - t <= z1_maxX && z1_maxX <= z2_maxX + t)) {
return IntervalRelations.FINISHES;
} else if (z1_minX <= z2_minX + t && (z2_maxX - t > z1_maxX && z1_maxX > z2_maxX + t)) {
return IntervalRelations.FINISHES_INVERSE;
} else if (z2_minX - t <= z1_minX && z1_minX <= z2_minX + t && (z2_maxX - t <= z1_maxX && z1_maxX <= z2_maxX + t)) {
return IntervalRelations.EQUALS;
} }
return intervalRelations;
return IntervalRelations.UNKNOWN;
} }
private static IntervalRelations getIntervalRelationY(Zone z1, Zone z2, double t) { private static List<IntervalRelations> getIntervalRelation(Pair<Double, Double> a, Pair<Double, Double> b, double t, boolean inverse) {
double z1_minY = z1.getY(); List<IntervalRelations> intervalRelations = new ArrayList<>();
double z1_maxY = z1_minY + z1.getHeight(); if (a.getRight() < b.getLeft() - t) {
double z2_minY = z2.getY(); intervalRelations.add(inverse ? IntervalRelations.PRECEDES_INVERSE : IntervalRelations.PRECEDES);
double z2_maxY = z2_minY + z2.getHeight(); } if (b.getLeft() - t <= a.getRight() && a.getRight() <= b.getLeft() + t) {
intervalRelations.add(inverse ? IntervalRelations.MEETS_INVERSE : IntervalRelations.MEETS);
// this is very wrong: check https://www.cs.rug.nl/~aiellom/publications/ijdarNoi.pdf } if (a.getLeft() < b.getLeft() - t && (b.getLeft() + t < a.getRight() && a.getRight() < b.getRight() - t)) {
if (z1_minY < z2_maxY - t) { intervalRelations.add(inverse ? IntervalRelations.OVERLAPS_INVERSE : IntervalRelations.OVERLAPS);
return IntervalRelations.PRECEDES_INVERSE; } if ((b.getLeft() - t <= a.getLeft() && a.getLeft() <= b.getLeft() + t) && a.getRight() < b.getRight() - t) {
} else if (z1_minY >= z2_maxY - t) { intervalRelations.add(inverse ? IntervalRelations.STARTS_INVERSE : IntervalRelations.STARTS);
return IntervalRelations.PRECEDES; } if (a.getLeft() > b.getLeft() + t && a.getRight() < b.getRight() + t) {
} else if (z2_maxY - t <= z1_minY && z1_minY <= z2_maxY + t) { intervalRelations.add(inverse ? IntervalRelations.DURING_INVERSE : IntervalRelations.DURING);
return IntervalRelations.MEETS_INVERSE; } if (a.getLeft() > b.getLeft() + t && (b.getRight() - t <= a.getRight() && a.getRight() <= b.getRight() + t)) {
} else if (z2_maxY - t > z1_minY && z1_minY > z2_maxY + t) { intervalRelations.add(inverse ? IntervalRelations.FINISHES_INVERSE : IntervalRelations.FINISHES);
return IntervalRelations.MEETS;
} else if (z1_maxY < z2_maxY - t && (z2_maxY + t < z1_minY && z1_minY < z2_minY - t)) {
return IntervalRelations.OVERLAPS_INVERSE;
} else if (z1_maxY >= z2_maxY - t && (z2_maxY + t >= z1_minY && z1_minY >= z2_minY - t)) {
return IntervalRelations.OVERLAPS;
} else if (z2_maxY - t <= z1_maxY && z1_maxY <= z2_maxY + t && z1_minY < z2_minY - t) {
return IntervalRelations.STARTS_INVERSE;
} else if (z2_maxY - t > z1_maxY && z1_maxY > z2_maxY + t && z1_minY >= z2_minY - t) {
return IntervalRelations.STARTS;
} else if (z1_maxY > z2_maxY + t && z1_minY < z2_minY - t) {
return IntervalRelations.DURING_INVERSE;
} else if (z1_maxY <= z2_maxY + t && z1_minY >= z2_minY - t) {
return IntervalRelations.DURING;
} else if (z1_maxY > z2_maxY + t && (z2_minY - t <= z1_minY && z1_minY <= z2_minY + t)) {
return IntervalRelations.FINISHES_INVERSE;
} else if (z1_maxY <= z2_maxY + t && (z2_minY - t > z1_minY && z1_minY > z2_minY + t)) {
return IntervalRelations.FINISHES;
} else if (z2_maxY - t <= z1_maxY && z1_maxY <= z2_maxY + t && (z2_minY - t <= z1_minY && z1_minY <= z2_minY + t)) {
return IntervalRelations.EQUALS;
} }
return intervalRelations;
return IntervalRelations.UNKNOWN;
} }
} }

2
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/service/ReadingOrderService.java
View File

@ -27,7 +27,7 @@ public class ReadingOrderService {
SpatialReasoningRules spatialReasoningRules = columnWise ? SpatialReasoningRules.COLUMN_WISE : SpatialReasoningRules.ROW_WISE; SpatialReasoningRules spatialReasoningRules = columnWise ? SpatialReasoningRules.COLUMN_WISE : SpatialReasoningRules.ROW_WISE;
var unsupervisedReadingOrderDetector = new UnsupervisedReadingOrderDetector(5, spatialReasoningRules, false); var unsupervisedReadingOrderDetector = new UnsupervisedReadingOrderDetector(1, spatialReasoningRules, false);
return unsupervisedReadingOrderDetector.get(zones); return unsupervisedReadingOrderDetector.get(zones);
} }

2
layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java
View File

@ -33,7 +33,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null); ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService); LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
Document document = buildGraph(fileName, LayoutParsingType.DOCSTRUM_XY); Document document = buildGraph(fileName, LayoutParsingType.DOCSTRUM);
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true); layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000); System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);