Compare commits

...
2 Commits
Author SHA1 Message Date
Dominique Eifländer 08e994d904 More refactoring 2024-02-20 14:04:28 +01:00
Dominique Eifländer 0c8c727303 Reading Order 2024-02-20 11:56:43 +01:00
17 changed files with 237 additions and 903 deletions
@@ -1,12 +1,17 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Character;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Line;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.LineBuilderService;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service.NearestNeighbourService;
@@ -20,6 +25,8 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class DocstrumSegmentationService {
private static final double MAX_VERTICAL_MERGE_DIST = 0.5;
private final NearestNeighbourService nearestNeighbourService;
private final SpacingService spacingService;
private final LineBuilderService lineBuilderService;
@@ -42,7 +49,105 @@ public class DocstrumSegmentationService {
var zones = zoneBuilderService.buildZones(lines, characterSpacing, lineSpacing);
return readingOrderService.resolve(zones);
zones = mergeLinesInZones(zones, characterSpacing, Double.NEGATIVE_INFINITY, 0.0, 0.0, lineSpacing * MAX_VERTICAL_MERGE_DIST);
return readingOrderService.resolve(zones, false);
}
// private List<Zone> mergeZones(List<Zone> zones, double tolerance) {
//
// List<BxBounds> bounds = new ArrayList<BxBounds>(zones.size());
// for (List<ComponentLine> zone : zones) {
// BxBoundsBuilder builder = new BxBoundsBuilder();
// for (ComponentLine line : zone) {
// for (Component component : line.getComponents()) {
// builder.expand(component.getChunk().getBounds());
// }
// }
// bounds.add(builder.getBounds());
// }
//
// List<List<ComponentLine>> outputZones = new ArrayList<List<ComponentLine>>();
// mainFor:
// for (int i = 0; i < zones.size(); i++) {
// for (int j = 0; j < zones.size(); j++) {
// if (i == j || bounds.get(j) == null || bounds.get(i) == null) {
// continue;
// }
// if (BxModelUtils.contains(bounds.get(j), bounds.get(i), tolerance)) {
// zones.get(j).addAll(zones.get(i));
// bounds.set(i, null);
// continue mainFor;
// }
// }
// outputZones.add(zones.get(i));
// }
// return outputZones;
// }
private List<Zone> mergeLinesInZones(List<Zone> zones,
double wordSpacing,
double minHorizontalDistance,
double maxHorizontalDistance,
double minVerticalDistance,
double maxVerticalDistance) {
List<Zone> outputZones = new ArrayList<>(zones.size());
for (Zone zone : zones) {
outputZones.add(mergeLinesInZone(zone, wordSpacing, minHorizontalDistance, maxHorizontalDistance, minVerticalDistance, maxVerticalDistance));
}
return outputZones;
}
private Zone mergeLinesInZone(Zone zone,
double wordSpacing,
double minHorizontalDistance,
double maxHorizontalDistance,
double minVerticalDistance,
double maxVerticalDistance) {
DisjointSets<Line> sets = new DisjointSets<>(zone.getLines());
for (int i = 0; i < zone.getLines().size(); i++) {
Line li = zone.getLines().get(i);
for (int j = i + 1; j < zone.getLines().size(); j++) {
Line lj = zone.getLines().get(j);
double hDist = li.horizontalDistance(lj);
double vDist = li.verticalDistance(lj);
if (minHorizontalDistance <= hDist && hDist <= maxHorizontalDistance && minVerticalDistance <= vDist && vDist <= maxVerticalDistance) {
sets.union(li, lj);
} else if (minVerticalDistance <= vDist && vDist <= maxVerticalDistance && Math.abs(hDist - Math.min(li.getLength(), lj.getLength())) < 0.1) {
boolean componentOverlap = false;
int overlappingCount = 0;
for (Character ci : li.getCharacters()) {
for (Character cj : lj.getCharacters()) {
double dist = ci.overlappingDistance(cj);
if (dist > 2) {
componentOverlap = true;
}
if (dist > 0) {
overlappingCount++;
}
}
}
if (!componentOverlap && overlappingCount <= 2) {
sets.union(li, lj);
}
}
}
}
List<Line> outputZone = new ArrayList<>();
for (Set<Line> group : sets) {
List<Character> components = new ArrayList<>();
for (Line line : group) {
components.addAll(line.getCharacters());
}
components.sort(Comparator.comparingDouble(Character::getX));
outputZone.add(new Line(components, wordSpacing));
}
return new Zone(outputZone);
}
}
@@ -1,30 +1,12 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
/**
* Filter class for neighbor objects that checks if the angle of the
* neighbor is within specified range.
*/
public abstract class AngleFilter {
public class AngleFilter {
private final double lowerAngle;
private final double upperAngle;
protected double lowerAngle;
protected double upperAngle;
private AngleFilter(double lowerAngle, double upperAngle) {
this.lowerAngle = lowerAngle;
this.upperAngle = upperAngle;
}
/**
* Constructs new angle filter.
*
* @param lowerAngle minimum angle in range [-3*pi/2, pi/2)
* @param upperAngle maximum angle in range [-pi/2, 3*pi/2)
* @return newly constructed angle filter
*/
public static AngleFilter newInstance(double lowerAngle, double upperAngle) {
public AngleFilter(double lowerAngle, double upperAngle) {
if (lowerAngle < -Math.PI / 2) {
lowerAngle += Math.PI;
@@ -32,59 +14,19 @@ public abstract class AngleFilter {
if (upperAngle >= Math.PI / 2) {
upperAngle -= Math.PI;
}
this.lowerAngle = lowerAngle;
this.upperAngle = upperAngle;
}
public boolean matches(Neighbor neighbor) {
if (lowerAngle <= upperAngle) {
return new AndFilter(lowerAngle, upperAngle);
return lowerAngle <= neighbor.getAngle() && neighbor.getAngle() < upperAngle;
} else {
return new OrFilter(lowerAngle, upperAngle);
return lowerAngle <= neighbor.getAngle() || neighbor.getAngle() < upperAngle;
}
}
public double getLowerAngle() {
return lowerAngle;
}
public double getUpperAngle() {
return upperAngle;
}
public abstract boolean matches(Neighbor neighbor);
public static final class AndFilter extends AngleFilter {
private AndFilter(double lowerAngle, double upperAngle) {
super(lowerAngle, upperAngle);
}
@Override
public boolean matches(Neighbor neighbor) {
return getLowerAngle() <= neighbor.getAngle() && neighbor.getAngle() < getUpperAngle();
}
}
public static final class OrFilter extends AngleFilter {
private OrFilter(double lowerAngle, double upperAngle) {
super(lowerAngle, upperAngle);
}
@Override
public boolean matches(Neighbor neighbor) {
return getLowerAngle() <= neighbor.getAngle() || neighbor.getAngle() < getUpperAngle();
}
}
}
@@ -1,6 +1,7 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
@@ -51,6 +52,20 @@ public class Character {
}
public double overlappingDistance(Character other) {
double[] xs = new double[4];
double s = Math.sin(-0), c = Math.cos(-0);
xs[0] = c * x - s * y;
xs[1] = c * (x + textPosition.getWidthDirAdj()) - s * (y + textPosition.getHeightDir());
xs[2] = c * other.x - s * other.y;
xs[3] = c * (other.x + other.textPosition.getWidthDirAdj()) - s * (other.y + other.textPosition.getHeightDir());
boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0];
Arrays.sort(xs);
return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1);
}
public void setNeighbors(List<Neighbor> neighbors) {
this.neighbors = neighbors;
@@ -10,41 +10,23 @@ import java.util.Set;
public class DisjointSets<E> implements Iterable<Set<E>> {
private final Map<E, Entry<E>> map = new HashMap<E, Entry<E>>();
private final Map<E, Entry<E>> map = new HashMap<>();
/**
* Constructs a new set of singletons.
*
* @param c elements of singleton sets
*/
public DisjointSets(Collection<? extends E> c) {
public DisjointSets(Collection<? extends E> collection) {
for (E element : c) {
for (E element : collection) {
map.put(element, new Entry<E>(element));
}
}
/**
* Checks if elements are in the same subsets.
*
* @param e1 element from a subset
* @param e2 element from a subset
* @return true if elements are in the same subset; false otherwise
*/
public boolean areTogether(E e1, E e2) {
return map.get(e1).findRepresentative() == map.get(e2).findRepresentative();
}
/**
* Merges subsets which elements e1 and e2 belong to.
*
* @param e1 element from a subset
* @param e2 element from a subset
*/
public void union(E e1, E e2) {
Entry<E> r1 = map.get(e1).findRepresentative();
@@ -62,7 +44,7 @@ public class DisjointSets<E> implements Iterable<Set<E>> {
@Override
public Iterator<Set<E>> iterator() {
return new Iterator<Set<E>>() {
return new Iterator<>() {
private final Iterator<Entry<E>> iterator = map.values().iterator();
private Entry<E> nextRepresentative;
@@ -1,33 +1,19 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model;
import java.util.Iterator;
import java.util.NoSuchElementException;
public class Histogram implements Iterable<Histogram.Bin> {
public class Histogram {
private static final double EPSILON = 1.0e-6;
private final double min;
private final double delta;
private final double resolution;
private double[] frequencies;
/**
* Constructs a new histogram for values in range [minValue, maxValue] with
* given resolution.
*
* @param minValue - minimum allowed value
* @param maxValue - maximum allowed value
* @param resolution - histogram's resolution
*/
public Histogram(double minValue, double maxValue, double resolution) {
this.min = minValue - EPSILON;
this.delta = maxValue - minValue + 2 * EPSILON;
double delta = maxValue - minValue + 2 * EPSILON;
int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution));
this.resolution = this.delta / size;
this.resolution = delta / size;
this.frequencies = new double[size];
}
@@ -47,25 +33,6 @@ public class Histogram implements Iterable<Histogram.Bin> {
}
public void circularKernelSmooth(double[] kernel) {
double[] newFrequencies = new double[frequencies.length];
int shift = (kernel.length - 1) / 2;
for (int i = 0; i < frequencies.length; i++) {
for (int d = 0; d < kernel.length; d++) {
int j = i + d - shift;
if (j < 0) {
j += frequencies.length;
} else if (j >= frequencies.length) {
j -= frequencies.length;
}
newFrequencies[i] += kernel[d] * frequencies[j];
}
}
frequencies = newFrequencies;
}
public double[] createGaussianKernel(double length, double stdDeviation) {
int r = (int) Math.round(length / resolution) / 2;
@@ -87,45 +54,24 @@ public class Histogram implements Iterable<Histogram.Bin> {
}
public void circularGaussianSmooth(double windowLength, double stdDeviation) {
circularKernelSmooth(createGaussianKernel(windowLength, stdDeviation));
}
public void gaussianSmooth(double windowLength, double stdDeviation) {
kernelSmooth(createGaussianKernel(windowLength, stdDeviation));
}
/**
* Adds single occurrence of given value to the histogram.
*
* @param value inserted values
*/
public void add(double value) {
frequencies[(int) ((value - min) / resolution)] += 1.0;
}
/**
* Returns histogram's number of bins.
*
* @return number of bins
*/
public int getSize() {
return frequencies.length;
}
/**
* Finds the histogram's peak value.
*
* @return peak value
*/
public double getPeakValue() {
int peakIndex = 0;
@@ -142,58 +88,4 @@ public class Histogram implements Iterable<Histogram.Bin> {
return ((double) peakIndex + peakEndIndex) / 2 * resolution + min;
}
@Override
public Iterator<Bin> iterator() {
return new Iterator() {
private int index = 0;
@Override
public boolean hasNext() {
return index < frequencies.length;
}
@Override
public Object next() {
if (index >= frequencies.length) {
throw new NoSuchElementException();
}
return new Bin(index++);
}
@Override
public void remove() {
throw new UnsupportedOperationException("Not supported yet.");
}
};
}
public final class Bin {
private final int index;
private Bin(int index) {
this.index = index;
}
public double getValue() {
return (index + 0.5) * resolution + min;
}
}
}
@@ -31,13 +31,13 @@ public class Line extends BoundingBox {
this.characters = characters;
if (characters.size() >= 2) {
// Simple linear regression
// linear regression
double sx = 0.0, sxx = 0.0, sxy = 0.0, sy = 0.0;
for (Character component : characters) {
sx += component.getX();
sxx += component.getX() * component.getX();
sxy += component.getX() * component.getY();
sy += component.getY();
for (Character character : characters) {
sx += character.getX();
sxx += character.getX() * character.getX();
sxy += character.getX() * character.getY();
sy += character.getY();
}
double b = (characters.size() * sxy - sx * sy) / (characters.size() * sxx - sx * sx);
double a = (sy - b * sx) / characters.size();
@@ -46,20 +46,18 @@ public class Line extends BoundingBox {
this.y0 = a + b * this.x0;
this.x1 = characters.get(characters.size() - 1).getX();
this.y1 = a + b * this.x1;
} else if (!characters.isEmpty()) {
Character component = characters.get(0);
double dx = component.getTextPosition().getWidthDirAdj() / 3;
double dy = dx * Math.tan(0);
this.x0 = component.getX() - dx;
this.x1 = component.getX() + dx;
this.y0 = component.getY() - dy;
this.y1 = component.getY() + dy;
} else {
throw new IllegalArgumentException("Component list must not be empty");
Character character = characters.get(0);
double dx = character.getTextPosition().getWidthDirAdj() / 3;
double dy = dx * Math.tan(0);
this.x0 = character.getX() - dx;
this.x1 = character.getX() + dx;
this.y0 = character.getY() - dy;
this.y1 = character.getY() + dy;
}
height = computeHeight();
computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
buildBox();
buildBBox();
}
@@ -136,7 +134,7 @@ public class Line extends BoundingBox {
}
private void buildBox() {
private void buildBBox() {
double minX = Double.POSITIVE_INFINITY;
double minY = Double.POSITIVE_INFINITY;
@@ -155,5 +153,13 @@ public class Line extends BoundingBox {
this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
}
public String toString() {
StringBuilder sb = new StringBuilder();
words.forEach(word -> sb.append(word.toString()).append(" "));
return sb.toString().trim();
}
}
@@ -16,11 +16,11 @@ public class Zone extends BoundingBox {
lines.sort(Comparator.comparingDouble(Line::getY));
this.lines = lines;
buildBox();
buildBBox();
}
public void buildBox() {
public void buildBBox() {
double minX = Double.POSITIVE_INFINITY;
double minY = Double.POSITIVE_INFINITY;
@@ -39,4 +39,12 @@ public class Zone extends BoundingBox {
this.setBBox(new Rectangle2D.Double(minX, minY, maxX - minX, maxY - minY));
}
public String toString() {
StringBuilder sb = new StringBuilder();
lines.forEach(line -> sb.append(line.toString()).append("\n"));
return sb.toString().trim();
}
}
@@ -1,64 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
import java.awt.geom.Rectangle2D;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
public class BoundingBoxZoneGroup extends BoundingBox {
private BoundingBox leftChild;
private BoundingBox rightChild;
public BoundingBoxZoneGroup(BoundingBox child1, BoundingBox child2) {
this.leftChild = child1;
this.rightChild = child2;
setBounds(Math.min(child1.getX(), child2.getX()),
Math.min(child1.getY(), child2.getY()),
Math.max(child1.getX() + child1.getWidth(), child2.getX() + child2.getWidth()),
Math.max(child1.getY() + child1.getHeight(), child2.getY() + child2.getHeight()));
}
public void setbBox(Rectangle2D bBox) {
super.setBBox(bBox);
}
public BoundingBox getLeftChild() {
return leftChild;
}
public BoundingBox getRightChild() {
return rightChild;
}
public BoundingBoxZoneGroup setLeftChild(BoundingBox obj) {
this.leftChild = obj;
return this;
}
public BoundingBoxZoneGroup setRightChild(BoundingBox obj) {
this.rightChild = obj;
return this;
}
public BoundingBoxZoneGroup setBounds(double x0, double y0, double x1, double y1) {
assert x1 >= x0;
assert y1 >= y0;
this.setBBox(new Rectangle2D.Double(x0, y0, x1 - x0, y1 - y0));
return this;
}
}
@@ -1,115 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
public class DistElem<E> implements Comparable<DistElem<E>> {
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (c ? 1231 : 1237);
long temp;
temp = Double.doubleToLongBits(dist);
result = prime * result + (int) (temp ^ (temp >>> 32));
result = prime * result + ((obj1 == null) ? 0 : obj1.hashCode());
result = prime * result + ((obj2 == null) ? 0 : obj2.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
DistElem other = (DistElem) obj;
if (c != other.c) {
return false;
}
if (Double.doubleToLongBits(dist) != Double.doubleToLongBits(other.dist)) {
return false;
}
if (obj1 == null) {
if (other.obj1 != null) {
return false;
}
} else if (!obj1.equals(other.obj1)) {
return false;
}
if (obj2 == null) {
if (other.obj2 != null) {
return false;
}
} else if (!obj2.equals(other.obj2)) {
return false;
}
return true;
}
boolean c;
double dist;
E obj1;
E obj2;
public boolean isC() {
return c;
}
public void setC(boolean c) {
this.c = c;
}
public double getDist() {
return dist;
}
public E getObj1() {
return obj1;
}
public E getObj2() {
return obj2;
}
public DistElem(boolean c, double dist, E obj1, E obj2) {
this.c = c;
this.dist = dist;
this.obj1 = obj1;
this.obj2 = obj2;
}
@Override
public int compareTo(DistElem<E> compareObject) {
double eps = 1E-3;
if (c == compareObject.c) {
return DoubleUtils.compareDouble(dist, compareObject.dist, eps);
} else {
return c ? -1 : 1;
}
}
}
@@ -1,258 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
import java.awt.geom.Rectangle2D;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
/**
* A set-like data structure for objects placed on a plane. Can efficiently find objects in a certain rectangular area.
* It maintains two parallel lists of objects, each of which is sorted by its x or y coordinate.
*
* @author Pawel Szostek
*/
public class DocumentPlane {
/**
* List of objects on the plane. Stored in a random order
*/
private final List<BoundingBox> objs;
/**
* Size of a grid square. If gridSize=50, then the plane is divided into squares of size 50. Each square contains
* objects placed in a 50x50 area
*/
private final int gridSize;
/**
* Redundant dictionary of objects on the plane. Allows efficient 2D space search. Keys are X-Y coordinates of a
* grid square. Single object can be stored under several keys (depending on its physical size). Grid squares are
* lazy-initialized.
*/
private final Map<GridXY, List<BoundingBox>> grid;
/**
* Representation of XY coordinates
*/
private static class GridXY {
public int x;
public int y;
public GridXY(int x, int y) {
this.x = x;
this.y = y;
}
@Override
public int hashCode() {
return x * y;
}
@Override
public boolean equals(Object obj) {
if (obj == null || getClass() != obj.getClass()) {
return false;
}
GridXY comparedObj = (GridXY) obj;
return x == comparedObj.x && y == comparedObj.y;
}
@Override
public String toString() {
return "(" + x + "," + y + ")";
}
}
public List<BoundingBox> getObjects() {
return objs;
}
public DocumentPlane(List<Zone> objectList, int gridSize) {
this.grid = new HashMap<GridXY, List<BoundingBox>>();
this.objs = new ArrayList<BoundingBox>();
this.gridSize = gridSize;
for (Zone obj : objectList) {
add(obj);
}
}
/**
* Looks for objects placed between obj1 and obj2 excluding them
*
* @param obj1 object
* @param obj2 object
* @return object list
*/
public List<BoundingBox> findObjectsBetween(BoundingBox obj1, BoundingBox obj2) {
double x0 = Math.min(obj1.getX(), obj2.getX());
double y0 = Math.min(obj1.getY(), obj2.getY());
double x1 = Math.max(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth());
double y1 = Math.max(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight());
assert x1 >= x0 && y1 >= y0;
Rectangle2D searchBounds = new Rectangle2D.Double(x0, y0, x1 - x0, y1 - y0);
List<BoundingBox> objsBetween = find(searchBounds);
/*
* the rectangle area must contain at least obj1 and obj2
*/
objsBetween.remove(obj1);
objsBetween.remove(obj2);
return objsBetween;
}
/**
* Checks if there is any object placed between obj1 and obj2
*
* @param obj1 object
* @param obj2 object
* @return true if anything is placed between, false otherwise
*/
public boolean anyObjectsBetween(BoundingBox obj1, BoundingBox obj2) {
List<BoundingBox> lObjs = findObjectsBetween(obj1, obj2);
return !(lObjs.isEmpty());
}
/**
* Adds object to the plane
*
* @param obj object
* @return document plane
*/
public DocumentPlane add(BoundingBox obj) {
int objsBefore = this.objs.size();
/*
* iterate over grid squares
*/
for (int y = ((int) obj.getY()) / gridSize; y <= ((int) (obj.getY() + obj.getHeight() + gridSize - 1)) / gridSize; ++y) {
for (int x = ((int) obj.getX()) / gridSize; x <= ((int) (obj.getX() + obj.getWidth() + gridSize - 1)) / gridSize; ++x) {
GridXY xy = new GridXY(x, y);
if (!grid.keySet().contains(xy)) {
/*
* add the non-existing key
*/
grid.put(xy, new ArrayList<BoundingBox>());
grid.get(xy).add(obj);
assert grid.get(xy).size() == 1;
} else {
grid.get(xy).add(obj);
}
}
}
objs.add(obj);
/*
* size of the object list should be incremented
*/
assert objsBefore + 1 == objs.size();
/*
* object list must contain the same number of objects as object dictionary
*/
assert objs.size() == elementsInGrid();
return this;
}
public DocumentPlane remove(BoundingBox obj) {
/*
* iterate over grid squares
*/
for (int y = ((int) obj.getY()) / gridSize; y <= ((int) (obj.getY() + obj.getHeight() + gridSize - 1)) / gridSize; ++y) {
for (int x = ((int) obj.getX()) / gridSize; x <= ((int) (obj.getX() + obj.getWidth() + gridSize - 1)) / gridSize; ++x) {
GridXY xy = new GridXY(x, y);
if (grid.get(xy).contains(obj)) {
grid.get(xy).remove(obj);
}
}
}
objs.remove(obj);
assert objs.size() == elementsInGrid();
return this;
}
/**
* Find objects within search bounds
*
* @param searchBounds is a search rectangle
* @return list of objects in!side search rectangle
*/
public List<BoundingBox> find(Rectangle2D searchBounds) {
List<BoundingBox> done = new ArrayList<BoundingBox>(); //contains already considered objects (wrt. optimization)
List<BoundingBox> ret = new ArrayList<BoundingBox>();
double x0 = searchBounds.getX();
double y0 = searchBounds.getY();
double y1 = searchBounds.getY() + searchBounds.getHeight();
double x1 = searchBounds.getX() + searchBounds.getWidth();
/*
* iterate over grid squares
*/
for (int y = (int) y0 / gridSize; y < ((int) (y1 + gridSize - 1)) / gridSize; ++y) {
for (int x = (int) x0 / gridSize; x < ((int) (x1 + gridSize - 1)) / gridSize; ++x) {
GridXY xy = new GridXY(x, y);
if (!grid.containsKey(xy)) {
continue;
}
for (BoundingBox obj : grid.get(xy)) {
if (done.contains(obj)) /*
* omit if already checked
*/ {
continue;
}
/*
* add to the checked objects
*/
done.add(obj);
/*
* check if two objects overlap
*/
if (obj.getX() + obj.getWidth() <= x0 || x1 <= obj.getX() || obj.getY() + obj.getHeight() <= y0 || y1 <= obj.getY()) {
continue;
}
ret.add(obj);
}
}
}
return ret;
}
/**
* Count objects stored in objects dictionary
*
* @return number of elements
*/
protected int elementsInGrid() {
List<BoundingBox> objs_ = new ArrayList<BoundingBox>();
for (GridXY coord : grid.keySet()) {
for (BoundingBox obj : grid.get(coord)) {
if (!objs_.contains(obj)) {
objs_.add(obj);
}
}
}
return objs_.size();
}
}
@@ -1,29 +0,0 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder;
import java.util.ArrayList;
import java.util.List;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
public class TreeToListConverter {
public List<Zone> convertToList(BoundingBoxZoneGroup obj) {
List<Zone> ret = new ArrayList<>();
if (obj.getLeftChild() instanceof Zone) {
Zone zone = (Zone) obj.getLeftChild();
ret.add(zone);
} else { // obj.getLeftChild() instanceof BxZoneGroup
ret.addAll(convertToList((BoundingBoxZoneGroup) obj.getLeftChild()));
}
if (obj.getRightChild() instanceof Zone) {
Zone zone = (Zone) obj.getRightChild();
ret.add(zone);
} else { // obj.getRightChild() instanceof BxZoneGroup
ret.addAll(convertToList((BoundingBoxZoneGroup) obj.getRightChild()));
}
return ret;
}
}
@@ -25,7 +25,7 @@ public class LineBuilderService {
double maxVerticalDistance = lineSpacing * MAX_VERTICAL_CHARACTER_DISTANCE;
DisjointSets<Character> sets = new DisjointSets<>(characters);
AngleFilter filter = AngleFilter.newInstance(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
AngleFilter filter = new AngleFilter(-ANGLE_TOLERANCE, ANGLE_TOLERANCE);
characters.forEach(character -> {
character.getNeighbors().forEach(neighbor -> {
@@ -1,243 +1,99 @@
package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.service;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.ListIterator;
import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.BoundingBox;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Zone;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.BoundingBoxZoneGroup;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.DistElem;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.DocumentPlane;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.TreeToListConverter;
import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.utils.DoubleUtils;
@Service
public class ReadingOrderService {
static final int GRIDSIZE = 50;
static final double EPS = 0.01;
static final int MAX_ZONES = 1000;
static final Comparator<BoundingBox> Y_ASCENDING_ORDER = new Comparator<BoundingBox>() {
private static final double THRESHOLD = 1;
@Override
public int compare(BoundingBox o1, BoundingBox o2) {
return DoubleUtils.compareDouble(o1.getY(), o2.getY(), EPS);
public List<Zone> resolve(List<Zone> zones, boolean yxOrder) {
if (zones.isEmpty() || zones.size() == 1) {
return zones;
}
};
static final Comparator<BoundingBox> X_ASCENDING_ORDER = new Comparator<BoundingBox>() {
@Override
public int compare(BoundingBox o1, BoundingBox o2) {
return DoubleUtils.compareDouble(o1.getX(), o2.getX(), EPS);
if (yxOrder) {
zones.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
.thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
return zones;
}
};
static final Comparator<BoundingBox> YX_ASCENDING_ORDER = new Comparator<BoundingBox>() {
@Override
public int compare(BoundingBox o1, BoundingBox o2) {
int yCompare = Y_ASCENDING_ORDER.compare(o1, o2);
return yCompare == 0 ? X_ASCENDING_ORDER.compare(o1, o2) : yCompare;
}
};
public List<Zone> resolve(List<Zone> zones) {
List<Zone> orderedZones;
if (zones.size() > MAX_ZONES) {
orderedZones = new ArrayList<>(zones);
Collections.sort(orderedZones, YX_ASCENDING_ORDER);
} else {
orderedZones = reorderZones(zones);
}
return orderedZones;
return resolveMultiColumnReadingOder(zones);
}
private List<Zone> reorderZones(List<Zone> unorderedZones) {
private List<Zone> resolveMultiColumnReadingOder(List<Zone> zones) {
if (unorderedZones.isEmpty()) {
return new ArrayList<>();
} else if (unorderedZones.size() == 1) {
List<Zone> ret = new ArrayList<>(1);
ret.add(unorderedZones.get(0));
return ret;
} else {
BoundingBoxZoneGroup bxZonesTree = groupZonesHierarchically(unorderedZones);
sortGroupedZones(bxZonesTree);
TreeToListConverter treeConverter = new TreeToListConverter();
List<Zone> orderedZones = treeConverter.convertToList(bxZonesTree);
assert unorderedZones.size() == orderedZones.size();
return orderedZones;
}
}
// Simple reading order resolver for multi column page layout as described here : https://pub.towardsai.net/advanced-rag-02-unveiling-pdf-parsing-b84ae866344e
// TODO implement a more fancy reading order resolver see https://github.com/BobLd/DocumentLayoutAnalysis/blob/master/README.md#reading-order
double minX = Double.POSITIVE_INFINITY;
double maxX = Double.NEGATIVE_INFINITY;
/**
* Builds a binary tree of zones and groups of zones from a list of unordered zones. This is done in hierarchical
* clustering by joining two least distant nodes. Distance is calculated in the distance() method.
*
* @param zones is a list of unordered zones
* @return root of the zones clustered in a tree
*/
private BoundingBoxZoneGroup groupZonesHierarchically(List<Zone> zones) {
/*
* Distance tuples are stored sorted by ascending distance value
*/
List<DistElem<BoundingBox>> dists = new ArrayList<DistElem<BoundingBox>>(zones.size() * zones.size() / 2);
for (int idx1 = 0; idx1 < zones.size(); ++idx1) {
for (int idx2 = idx1 + 1; idx2 < zones.size(); ++idx2) {
Zone zone1 = zones.get(idx1);
Zone zone2 = zones.get(idx2);
dists.add(new DistElem<BoundingBox>(false, distance(zone1, zone2), zone1, zone2));
for (Zone zone : zones) {
if (zone.getX() < minX) {
minX = zone.getX();
}
if (zone.getX() + zone.getWidth() > maxX) {
maxX = zone.getX() + zone.getWidth();
}
}
Collections.sort(dists);
DocumentPlane plane = new DocumentPlane(zones, GRIDSIZE);
while (!dists.isEmpty()) {
DistElem<BoundingBox> distElem = dists.get(0);
dists.remove(0);
if (!distElem.isC() && plane.anyObjectsBetween(distElem.getObj1(), distElem.getObj2())) {
dists.add(new DistElem<BoundingBox>(true, distElem.getDist(), distElem.getObj1(), distElem.getObj2()));
continue;
}
BoundingBoxZoneGroup newGroup = new BoundingBoxZoneGroup(distElem.getObj1(), distElem.getObj2());
plane.remove(distElem.getObj1()).remove(distElem.getObj2());
dists = removeDistElementsContainingObject(dists, distElem.getObj1());
dists = removeDistElementsContainingObject(dists, distElem.getObj2());
for (BoundingBox other : plane.getObjects()) {
dists.add(new DistElem<BoundingBox>(false, distance(other, newGroup), newGroup, other));
}
Collections.sort(dists);
plane.add(newGroup);
}
assert plane.getObjects().size() == 1 : "There should be one object left at the plane after grouping";
return (BoundingBoxZoneGroup) plane.getObjects().get(0);
}
double midLineXCoordinate = (minX + maxX) / 2;
/**
* Removes all distance tuples containing obj
*/
private List<DistElem<BoundingBox>> removeDistElementsContainingObject(Collection<DistElem<BoundingBox>> list, BoundingBox obj) {
List<DistElem<BoundingBox>> ret = new ArrayList<DistElem<BoundingBox>>();
for (DistElem<BoundingBox> distElem : list) {
if (distElem.getObj1() != obj && distElem.getObj2() != obj) {
ret.add(distElem);
List<Zone> leftOf = new ArrayList<>();
List<Zone> rightOf = new ArrayList<>();
List<Zone> middle = new ArrayList<>();
for (Zone zone : zones) {
if (zone.getX() < midLineXCoordinate && zone.getX() + zone.getWidth() < midLineXCoordinate) {
leftOf.add(zone);
} else if (zone.getX() > midLineXCoordinate && zone.getX() + zone.getWidth() > midLineXCoordinate) {
rightOf.add(zone);
} else {
middle.add(zone);
}
}
return ret;
}
leftOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
.thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
/**
* Swaps children of BxZoneGroup if necessary. A group with smaller sort factor is placed to the left (leftChild).
* An object with greater sort factor is placed on the right (rightChild). This plays an important role when
* traversing the tree in conversion to a one dimensional list.
*
* @param group
*/
private void sortGroupedZones(BoundingBoxZoneGroup group) {
rightOf.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
.thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
BoundingBox leftChild = group.getLeftChild();
BoundingBox rightChild = group.getRightChild();
if (shouldBeSwapped(leftChild, rightChild)) {
// swap
group.setLeftChild(rightChild);
group.setRightChild(leftChild);
middle.sort(Comparator.comparing(BoundingBox::getY, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD))
.thenComparing(BoundingBox::getX, (o1, o2) -> DoubleUtils.compareDouble(o1, o2, THRESHOLD)));
List<Zone> sortedZones = new ArrayList<>();
sortedZones.addAll(leftOf);
sortedZones.addAll(rightOf);
ListIterator<Zone> itty = middle.listIterator();
while (itty.hasNext()) {
Zone current = itty.next();
for (int i = 0; i < sortedZones.size(); i++) {
if (current.getY() < sortedZones.get(i).getY()) {
sortedZones.add(i, current);
itty.remove();
break;
}
}
}
if (leftChild instanceof BoundingBoxZoneGroup) // if the child is a tree node, then recurse
{
sortGroupedZones((BoundingBoxZoneGroup) leftChild);
}
if (rightChild instanceof BoundingBoxZoneGroup) // as above - recurse
{
sortGroupedZones((BoundingBoxZoneGroup) rightChild);
}
}
sortedZones.addAll(middle);
private boolean shouldBeSwapped(BoundingBox first, BoundingBox second) {
double cx, cy, cw, ch, ox, oy, ow, oh;
cx = first.getBBox().getX();
cy = first.getBBox().getY();
cw = first.getBBox().getWidth();
ch = first.getBBox().getHeight();
ox = second.getBBox().getX();
oy = second.getBBox().getY();
ow = second.getBBox().getWidth();
oh = second.getBBox().getHeight();
// Determine Octant
//
// 0 | 1 | 2
// __|___|__
// 7 | 9 | 3 First is placed in 9th square
// __|___|__
// 6 | 5 | 4
if (cx + cw <= ox) { //2,3,4
return false;
} else if (ox + ow <= cx) { //0,6,7
return true; //6
} else if (cy + ch <= oy) {
return false; //5
} else if (oy + oh <= cy) {
return true; //1
} else { //two zones
double xdiff = ox + ow / 2 - cx - cw / 2;
double ydiff = oy + oh / 2 - cy - ch / 2;
return xdiff + ydiff < 0;
}
}
/**
* A distance function between two TextBoxes.
* <p>
* Consider the bounding rectangle for obj1 and obj2. Return its area minus the areas of obj1 and obj2, shown as
* 'www' below. This value may be negative. (x0,y0) +------+..........+ | obj1 |wwwwwwwwww: +------+www+------+
* :wwwwwwwwww| obj2 | +..........+------+ (x1,y1)
*
* @return distance value based on objects' coordinates and physical size on a plane
*/
private double distance(BoundingBox obj1, BoundingBox obj2) {
double x0 = Math.min(obj1.getX(), obj2.getX());
double y0 = Math.min(obj1.getY(), obj2.getY());
double x1 = Math.max(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth());
double y1 = Math.max(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight());
double dist = ((x1 - x0) * (y1 - y0) - obj1.getArea() - obj2.getArea());
double obj1X = obj1.getX();
double obj1CenterX = obj1.getX() + obj1.getWidth() / 2;
double obj1CenterY = obj1.getY() + obj1.getHeight() / 2;
double obj2X = obj2.getX();
double obj2CenterX = obj2.getX() + obj2.getWidth() / 2;
double obj2CenterY = obj2.getY() + obj2.getHeight() / 2;
double obj1obj2VectorCosineAbsLeft = Math.abs((obj2X - obj1X) / Math.sqrt((obj2X - obj1X) * (obj2X - obj1X) + (obj2CenterY - obj1CenterY) * (obj2CenterY - obj1CenterY)));
double obj1obj2VectorCosineAbsCenter = Math.abs((obj2CenterX - obj1CenterX) / Math.sqrt((obj2CenterX - obj1CenterX) * (obj2CenterX - obj1CenterX) + (obj2CenterY - obj1CenterY) * (obj2CenterY - obj1CenterY)));
double cosine = Math.min(obj1obj2VectorCosineAbsLeft, obj1obj2VectorCosineAbsCenter);
final double MAGIC_COEFF = 0.5;
return dist * (MAGIC_COEFF + cosine);
return sortedZones;
}
}
@@ -40,7 +40,7 @@ public class SpacingService {
}
}
Histogram histogram = new Histogram(0, maxDistance, SPACING_HISTOGRAM_RESOLUTION);
AngleFilter filter = AngleFilter.newInstance(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
AngleFilter filter = new AngleFilter(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE);
for (Character component : components) {
for (Neighbor neighbor : component.getNeighbors()) {
if (filter.matches(neighbor)) {
@@ -41,6 +41,7 @@ public class ZoneBuilderService {
lines.forEach(outerLine -> //
lines.forEach(innerLine -> {
double scale = Math.min(outerLine.getHeight(), innerLine.getHeight()) / meanHeight;
scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE));
@@ -49,13 +50,8 @@ public class ZoneBuilderService {
double horizontalDistance = outerLine.horizontalDistance(innerLine) / scale;
double verticalDistance = outerLine.verticalDistance(innerLine) / scale;
// Line over or above
if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance) {
sets.union(outerLine, innerLine);
}
// Split line that needs later merging
else if (minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
if (minHorizontalDistance <= horizontalDistance && verticalDistance <= maxVerticalDistance //
|| minHorizontalMergeDistance <= horizontalDistance && verticalDistance <= maxVerticalMergeDistance) {
sets.union(outerLine, innerLine);
}
}
@@ -25,8 +25,6 @@ public class ViewerDocumentTest extends BuildDocumentTest {
@SneakyThrows
public void testViewerDocument() {
System.out.println("<<<<<<<<<<" + Math.sin(-0) + "aaa " + Math.cos(0) + Math.tan(0));
String fileName = "files/Plenarprotokoll 1 (keine Druchsache!) (1).pdf";
String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";