diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java index 01b4cdf..95ce4c6 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.apache.pdfbox.Loader; @@ -26,6 +27,7 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument; import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage; +import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationSection; import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document; import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage; import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings; @@ -47,6 +49,8 @@ import com.knecon.fforesight.service.layoutparser.processor.services.blockificat import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService; import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService; import com.knecon.fforesight.service.layoutparser.processor.services.classification.TaasClassificationService; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.DocstrumSegmenter; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.HierarchicalReadingOrderResolver; import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory; import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper; import com.knecon.fforesight.service.layoutparser.processor.services.mapper.TaasDocumentDataMapper; @@ -86,6 +90,8 @@ public class LayoutParsingPipeline { RedactManagerBlockificationService redactManagerBlockificationService; LayoutGridService layoutGridService; ObservationRegistry observationRegistry; + // DocstrumSegmenter docstrumSegmenter; + HierarchicalReadingOrderResolver hierarchicalReadingOrderResolver; public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException { @@ -243,11 +249,38 @@ public class LayoutParsingPipeline { PDRectangle cropbox = pdPage.getCropBox(); CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings()); - ClassificationPage classificationPage = switch (layoutParsingType) { - case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); - case TAAS -> taasBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); - case DOCUMINE -> docuMineBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); - }; + // Docstrum + AtomicInteger num = new AtomicInteger(pageNumber); + var zones = new DocstrumSegmenter().segmentPage(stripper.getTextPositionSequences()); + zones = hierarchicalReadingOrderResolver.resolve(zones); + + List pageBlocks = new ArrayList<>(); + AtomicInteger numOnPage = new AtomicInteger(1); +// List textPositionSequences = new ArrayList<>(); + zones.forEach(zone -> { + + List textPositionSequences = new ArrayList<>(); + zone.getLines().forEach(line -> { + line.getWords().forEach(word -> { + textPositionSequences.add(new TextPositionSequence(word.getTextPositions(), num.get())); + }); + }); + + var cps = redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical()); + + cps.getTextBlocks().forEach(cp -> { + pageBlocks.add(redactManagerBlockificationService.buildTextBlock(((TextPageBlock) cp).getSequences(), numOnPage.getAndIncrement())); + }); + }); + +// ClassificationPage classificationPage = switch (layoutParsingType) { +// case REDACT_MANAGER -> redactManagerBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical()); +// case TAAS -> taasBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical()); +// case DOCUMINE -> docuMineBlockificationService.blockify(textPositionSequences, cleanRulings.getHorizontal(), cleanRulings.getVertical()); +// }; + + ClassificationPage classificationPage = new ClassificationPage(pageBlocks); + classificationPage.setCleanRulings(cleanRulings); classificationPage.setRotation(rotation); classificationPage.setLandscape(isLandscape); @@ -283,9 +316,19 @@ public class LayoutParsingPipeline { case REDACT_MANAGER -> redactManagerClassificationService.classifyDocument(classificationDocument); } + List sections = new ArrayList<>(); + for (var page : classificationPages) { + page.getTextBlocks().forEach(block -> { + block.setPage(page.getPageNumber()); + var section = sectionsBuilderService.buildTextBlock(List.of(block), "a"); + sections.add(section); + }); + } + classificationDocument.setSections(sections); + log.info("Building Sections for {}", identifier); - sectionsBuilderService.buildSections(classificationDocument); - sectionsBuilderService.addImagesToSections(classificationDocument); +// sectionsBuilderService.buildSections(classificationDocument); +// sectionsBuilderService.addImagesToSections(classificationDocument); return classificationDocument; } diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java index a654636..36c3081 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationPage.java @@ -12,15 +12,16 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRul import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter; import lombok.Data; +import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; -import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent; @Data @RequiredArgsConstructor public class ClassificationPage { @NonNull + @Getter private List textBlocks; private List images = new ArrayList<>(); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java index ccea113..ae6c9e3 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java @@ -45,6 +45,9 @@ public class RedTextPosition { @JsonIgnore private String fontName; + @JsonIgnore + private RedTextPosition parent; + @SneakyThrows public static RedTextPosition fromTextPosition(TextPosition textPosition) { diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java index 0442af6..0bc5ac0 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java @@ -17,6 +17,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.NoArgsConstructor; @EqualsAndHashCode(callSuper = true) @@ -27,6 +28,7 @@ import lombok.NoArgsConstructor; public class TextPageBlock extends AbstractPageBlock { @Builder.Default + @Getter private List sequences = new ArrayList<>(); @JsonIgnore @@ -73,7 +75,7 @@ public class TextPageBlock extends AbstractPageBlock { return sequences.get(0).getPageWidth(); } - + public static TextPageBlock merge(List textBlocksToMerge) { @@ -82,6 +84,7 @@ public class TextPageBlock extends AbstractPageBlock { return fromTextPositionSequences(sequences); } + public static TextPageBlock fromTextPositionSequences(List wordBlockList) { TextPageBlock textBlock = null; @@ -133,7 +136,6 @@ public class TextPageBlock extends AbstractPageBlock { } - /** * Returns the minX value in pdf coordinate system. * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation. diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java index 82829c6..4977bd1 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java @@ -55,6 +55,18 @@ public class TextPositionSequence implements CharSequence { } + public TextPositionSequence(List textPositions, int page) { + + this.textPositions = textPositions; + this.page = page; + this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir()); + this.rotation = textPositions.get(0).getRotation(); + this.pageHeight = textPositions.get(0).getPageHeight(); + this.pageWidth = textPositions.get(0).getPageWidth(); + this.isParagraphStart = false; + } + + @Override public int length() { diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionsBuilderService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionsBuilderService.java index 04cc930..ae8be2a 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionsBuilderService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionsBuilderService.java @@ -240,7 +240,7 @@ public class SectionsBuilderService { } - private ClassificationSection buildTextBlock(List wordBlockList, String lastHeadline) { + public ClassificationSection buildTextBlock(List wordBlockList, String lastHeadline) { ClassificationSection section = new ClassificationSection(); diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java index 3062c78..bf3b966 100644 --- a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/RedactManagerBlockificationService.java @@ -57,7 +57,7 @@ public class RedactManagerBlockificationService { boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines); boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir()); - if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) { + if (prev != null && (splitByDir || isSplitByRuling)) { Orientation prevOrientation = null; if (!chunkBlockList.isEmpty()) { @@ -167,7 +167,7 @@ public class RedactManagerBlockificationService { } - private TextPageBlock buildTextBlock(List wordBlockList, int indexOnPage) { + public TextPageBlock buildTextBlock(List wordBlockList, int indexOnPage) { TextPageBlock textBlock = null; diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/AngleFilter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/AngleFilter.java new file mode 100644 index 0000000..e1a9fb7 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/AngleFilter.java @@ -0,0 +1,92 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum; + +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.docstrum.Neighbor; + +/** + * Filter class for neighbor objects that checks if the angle of the + * neighbor is within specified range. + */ +public abstract class AngleFilter { + + private final double lowerAngle; + private final double upperAngle; + + + private AngleFilter(double lowerAngle, double upperAngle) { + + this.lowerAngle = lowerAngle; + this.upperAngle = upperAngle; + } + + + /** + * Constructs new angle filter. + * + * @param lowerAngle minimum angle in range [-3*pi/2, pi/2) + * @param upperAngle maximum angle in range [-pi/2, 3*pi/2) + * @return newly constructed angle filter + */ + public static AngleFilter newInstance(double lowerAngle, double upperAngle) { + + if (lowerAngle < -Math.PI / 2) { + lowerAngle += Math.PI; + } + if (upperAngle >= Math.PI / 2) { + upperAngle -= Math.PI; + } + if (lowerAngle <= upperAngle) { + return new AndFilter(lowerAngle, upperAngle); + } else { + return new OrFilter(lowerAngle, upperAngle); + } + } + + + public double getLowerAngle() { + + return lowerAngle; + } + + + public double getUpperAngle() { + + return upperAngle; + } + + + public abstract boolean matches(Neighbor neighbor); + + + public static final class AndFilter extends AngleFilter { + + private AndFilter(double lowerAngle, double upperAngle) { + + super(lowerAngle, upperAngle); + } + + + @Override + public boolean matches(Neighbor neighbor) { + + return getLowerAngle() <= neighbor.getAngle() && neighbor.getAngle() < getUpperAngle(); + } + + } + + public static final class OrFilter extends AngleFilter { + + private OrFilter(double lowerAngle, double upperAngle) { + + super(lowerAngle, upperAngle); + } + + + @Override + public boolean matches(Neighbor neighbor) { + + return getLowerAngle() <= neighbor.getAngle() || neighbor.getAngle() < getUpperAngle(); + } + + } + +} \ No newline at end of file diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/DocstrumSegmenter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/DocstrumSegmenter.java new file mode 100644 index 0000000..12ed4a7 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/DocstrumSegmenter.java @@ -0,0 +1,767 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.google.common.collect.Lists; +import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.DisjointSets; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.Histogram; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.BoundingBox; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Line; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Word; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Zone; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.docstrum.Character; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.docstrum.Neighbor; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.utils.BoundingBoxBuilder; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.utils.ZoneUtils; + +@Service +public class DocstrumSegmenter { + + public static final int MAX_ZONES_PER_PAGE = 300; + public static final double ORIENTATION_MARGIN = 0.2; + public static final int LINES_PER_PAGE_MARGIN = 100; + + private static final double DISTANCE_STEP = 16.0; + + /** + * Angle histogram resolution in radians per bin. + */ + private static final double ANGLE_HIST_RESOLUTION = Math.toRadians(0.5); + + /** + * Angle histogram smoothing window length in radians. + * Length of angle histogram is equal to pi. + */ + private static final double ANGLE_HIST_SMOOTHING_LEN = 0.25 * Math.PI; + + /** + * Angle histogram gaussian smoothing window standard deviation in radians. + */ + private static final double ANGLE_HIST_SMOOTHING_STDDEV = 0.0625 * Math.PI; + + /** + * Spacing histogram resolution per bin. + */ + private static final double SPACING_HIST_RESOLUTION = 0.5; + + /** + * Spacing histogram smoothing window length. + */ + private static final double SPACING_HIST_SMOOTHING_LEN = 2.5; + + /** + * Spacing histogram gaussian smoothing window standard deviation. + */ + private static final double SPACING_HIST_SMOOTHING_STDDEV = 0.5; + + /** + * Maximum vertical component distance multiplier used during line + * determination. + *

+ * Maximum vertical distance between components (characters) that belong + * to the same line is equal to the product of this value and estimated + * between-line spacing. + */ + private static final double MAX_VERTICAL_COMP_DIST = 0.67; + + /** + * Minimum line size scale value. + *

+ * During zone determination (merging lines into zones) line height is + * taken into account. To achieve this, line size scale is estimated and + * limited to range [minLineSizeScale, maxLineSizeScale]. + */ + private static final double MIN_LINE_SIZE_SCALE = 0.9; + + /** + * Maximum line size scale value. + *

+ * See minLineSizeScale for more information. + */ + private static final double MAX_LINE_SIZE_SCALE = 2.5; + + /** + * Minimum horizontal line distance multiplier. + *

+ * Minimum horizontal distance between lines that belong to the same zone + * is equal to the product of this value and estimated within-line spacing. + */ + private static final double MIN_HORIZONTAL_DIST = -0.5; + + /** + * Minimum vertical line distance multiplier. + *

+ * Minimum vertical distance between lines that belong to the same zone + * is equal to the product of this value and estimated between-line spacing. + */ + private static final double MIN_VERTICAL_DIST = 0.0; + + /** + * Maximum vertical line distance multiplier. + *

+ * Maximum vertical distance between lines that belong to the same zone + * is equal to the product of this value and estimated between-line spacing. + */ + private static final double MAX_VERTICAL_DIST = 1.2; + + /** + * Component distance character spacing multiplier. + *

+ * Maximum distance between components that belong to the same line is + * equal to (lineSpacing * componentDistanceLineMultiplier + + * characterSpacing * componentDistanceCharacterMultiplier), where + * lineSpacing and characterSpacing are estimated between-line and + * within-line spacing, respectively. + */ + private static final double COMP_DIST_CHAR = 3.5; + + /** + * Word distance multiplier. + *

+ * Maximum distance between components that belong to the same word is + * equal to the product of this value and estimated within-line spacing. + */ + private static final double WORD_DIST_MULT = 0.2; + + /** + * Minimum horizontal line merge distance multiplier. + *

+ * Minimum horizontal distance between lines that should be merged is equal + * to the product of this value and estimated within-line spacing. + *

+ * Because split lines do not overlap this value should be negative. + */ + + private static final double MIN_HORIZONTAL_MERGE_DIST = -3.0; + + /** + * Maximum vertical line merge distance multiplier. + *

+ * Maximum vertical distance between lines that should be merged is equal + * to the product of this value and estimated between-line spacing. + */ + + private static final double MAX_VERTICAL_MERGE_DIST = 0.5; + + /** + * Angle tolerance for comparisons of angles between components and angles + * between lines. + */ + private static final double ANGLE_TOLERANCE = Math.PI / 6; + + /** + * Number of nearest-neighbors found per component. + */ + private static final int NEIGHBOUR_COUNT = 8; + + + public List segmentPage(List textPositions) { + + var positions = textPositions.stream().map(t -> t.getTextPositions()).flatMap(List::stream).collect(Collectors.toList()); + + var components = positions.stream().map(chunk -> new Character(chunk)).collect(Collectors.toList()); + + Character[] componentsArray = new Character[positions.size()]; + components.toArray(componentsArray); + + Arrays.sort(componentsArray, Character.CharacterXComparator.getInstance()); + findNeighbors(componentsArray); + + double orientation = computeInitialOrientation(components); + + double characterSpacing = computeCharacterSpacing(components, orientation); + double lineSpacing = computeLineSpacing(components, orientation); + + List lines = determineLines(components, orientation, characterSpacing * COMP_DIST_CHAR, lineSpacing * MAX_VERTICAL_COMP_DIST); + + if (Math.abs(orientation) > ORIENTATION_MARGIN) { + List linesZero = determineLines(components, 0, characterSpacing * COMP_DIST_CHAR, lineSpacing * MAX_VERTICAL_COMP_DIST); + + if (Math.abs(lines.size() - LINES_PER_PAGE_MARGIN) > Math.abs(linesZero.size() - LINES_PER_PAGE_MARGIN)) { + orientation = 0; + lines = linesZero; + } + } + + double lineOrientation = computeOrientation(lines); + if (!Double.isNaN(lineOrientation)) { + orientation = lineOrientation; + } + + List> zones = determineZones(lines, + orientation, + characterSpacing * MIN_HORIZONTAL_DIST, + Double.POSITIVE_INFINITY, + lineSpacing * MIN_VERTICAL_DIST, + lineSpacing * MAX_VERTICAL_DIST, + characterSpacing * MIN_HORIZONTAL_MERGE_DIST, + 0.0, + 0.0, + lineSpacing * MAX_VERTICAL_MERGE_DIST); + zones = mergeZones(zones, characterSpacing * 0.5); + zones = mergeLines(zones, orientation, Double.NEGATIVE_INFINITY, 0.0, 0.0, lineSpacing * MAX_VERTICAL_MERGE_DIST); + return convertToBxModel(zones, WORD_DIST_MULT * characterSpacing); + } + + + /** + * Performs for each component search for nearest-neighbors and stores the + * result in component's neighbors attribute. + * + * @param components array of components + * equal to the number of nearest-neighbors per component. + */ + private void findNeighbors(Character[] components) { + + if (components.length == 0) { + return; + } + if (components.length == 1) { + components[0].setNeighbors(new ArrayList()); + return; + } + int pageNeighborCount = NEIGHBOUR_COUNT; + if (components.length <= NEIGHBOUR_COUNT) { + pageNeighborCount = components.length - 1; + } + + List candidates = new ArrayList(); + for (int i = 0; i < components.length; i++) { + int start = i, end = i + 1; + // Contains components from components array + // from ranges [start, i) and [i+1, end) + double dist = Double.POSITIVE_INFINITY; + for (double searchDist = 0; searchDist < dist; ) { + searchDist += DISTANCE_STEP; + boolean newCandidatesFound = false; + + while (start > 0 && components[i].getX() - components[start - 1].getX() < searchDist) { + start--; + candidates.add(new Neighbor(components[start], components[i])); + if (candidates.size() > pageNeighborCount) { + Collections.sort(candidates, NeighborDistanceComparator.getInstance()); + candidates.subList(pageNeighborCount, candidates.size()).clear(); + } + newCandidatesFound = true; + } + while (end < components.length && components[end].getX() - components[i].getX() < searchDist) { + candidates.add(new Neighbor(components[end], components[i])); + if (candidates.size() > pageNeighborCount) { + Collections.sort(candidates, NeighborDistanceComparator.getInstance()); + candidates.subList(pageNeighborCount, candidates.size()).clear(); + } + end++; + newCandidatesFound = true; + } + + if (newCandidatesFound && candidates.size() >= pageNeighborCount) { + Collections.sort(candidates, NeighborDistanceComparator.getInstance()); + dist = candidates.get(pageNeighborCount - 1).getDistance(); + } + } + candidates.subList(pageNeighborCount, candidates.size()).clear(); + components[i].setNeighbors(new ArrayList(candidates)); + candidates.clear(); + } + } + + + /** + * Computes initial orientation estimation based on nearest-neighbors' angles. + * + * @param components + * @return initial orientation estimation + */ + private double computeInitialOrientation(List components) { + + Histogram histogram = new Histogram(-Math.PI / 2, Math.PI / 2, ANGLE_HIST_RESOLUTION); + for (Character component : components) { + for (Neighbor neighbor : component.getNeighbors()) { + histogram.add(neighbor.getAngle()); + } + } + // Rectangular smoothing window has been replaced with gaussian smoothing window + histogram.circularGaussianSmooth(ANGLE_HIST_SMOOTHING_LEN, ANGLE_HIST_SMOOTHING_STDDEV); + return histogram.getPeakValue(); + } + + + /** + * Computes within-line spacing based on nearest-neighbors distances. + * + * @param components + * @param orientation estimated text orientation + * @return estimated within-line spacing + */ + private double computeCharacterSpacing(List components, double orientation) { + + return computeSpacing(components, orientation); + } + + + /** + * Computes between-line spacing based on nearest-neighbors distances. + * + * @param components + * @param orientation estimated text orientation + * @return estimated between-line spacing + */ + private double computeLineSpacing(List components, double orientation) { + + if (orientation >= 0) { + return computeSpacing(components, orientation - Math.PI / 2); + } else { + return computeSpacing(components, orientation + Math.PI / 2); + } + } + + + private double computeSpacing(List components, double angle) { + + double maxDistance = Double.NEGATIVE_INFINITY; + for (Character component : components) { + for (Neighbor neighbor : component.getNeighbors()) { + maxDistance = Math.max(maxDistance, neighbor.getDistance()); + } + } + Histogram histogram = new Histogram(0, maxDistance, SPACING_HIST_RESOLUTION); + AngleFilter filter = AngleFilter.newInstance(angle - ANGLE_TOLERANCE, angle + ANGLE_TOLERANCE); + for (Character component : components) { + for (Neighbor neighbor : component.getNeighbors()) { + if (filter.matches(neighbor)) { + histogram.add(neighbor.getDistance()); + } + } + } + // Rectangular smoothing window has been replaced with gaussian smoothing window + histogram.gaussianSmooth(SPACING_HIST_SMOOTHING_LEN, SPACING_HIST_SMOOTHING_STDDEV); + return histogram.getPeakValue(); + } + + + /** + * Groups components into text lines. + * + * @param components component list + * @param orientation - estimated text orientation + * @param maxHorizontalDistance - maximum horizontal distance between components + * @param maxVerticalDistance - maximum vertical distance between components + * @return lines of components + */ + private List determineLines(List components, double orientation, double maxHorizontalDistance, double maxVerticalDistance) { + + DisjointSets sets = new DisjointSets(components); + AngleFilter filter = AngleFilter.newInstance(orientation - ANGLE_TOLERANCE, orientation + ANGLE_TOLERANCE); + for (Character component : components) { + for (Neighbor neighbor : component.getNeighbors()) { + double x = neighbor.getHorizontalDistance() / maxHorizontalDistance; + double y = neighbor.getVerticalDistance() / maxVerticalDistance; + if (filter.matches(neighbor) && x * x + y * y <= 1) { + sets.union(component, neighbor.getCharacter()); + } + } + } + List lines = new ArrayList(); + for (Set group : sets) { + List lineComponents = new ArrayList(group); + Collections.sort(lineComponents, Character.CharacterXComparator.getInstance()); + lines.add(new ComponentLine(lineComponents, orientation)); + } + return lines; + } + + + private double computeOrientation(List lines) { + // Compute weighted mean of line angles + double valueSum = 0.0; + double weightSum = 0.0; + for (ComponentLine line : lines) { + valueSum += line.getAngle() * line.getLength(); + weightSum += line.getLength(); + } + return valueSum / weightSum; + } + + + private List> determineZones(List lines, + double orientation, + double minHorizontalDistance, + double maxHorizontalDistance, + double minVerticalDistance, + double maxVerticalDistance, + double minHorizontalMergeDistance, + double maxHorizontalMergeDistance, + double minVerticalMergeDistance, + double maxVerticalMergeDistance) { + + DisjointSets sets = new DisjointSets(lines); + // Mean height is computed so that all distances can be scaled + // relative to the line height + double meanHeight = 0.0, weights = 0.0; + for (ComponentLine line : lines) { + double weight = line.getLength(); + meanHeight += line.getHeight() * weight; + weights += weight; + } + meanHeight /= weights; + + for (int i = 0; i < lines.size(); i++) { + ComponentLine li = lines.get(i); + for (int j = i + 1; j < lines.size(); j++) { + ComponentLine lj = lines.get(j); + double scale = Math.min(li.getHeight(), lj.getHeight()) / meanHeight; + scale = Math.max(MIN_LINE_SIZE_SCALE, Math.min(scale, MAX_LINE_SIZE_SCALE)); + // "<=" is used instead of "<" for consistency and to allow setting minVertical(Merge)Distance + // to 0.0 with meaning "no minimal distance required" + if (!sets.areTogether(li, lj) && li.angularDifference(lj) <= ANGLE_TOLERANCE) { + double hDist = li.horizontalDistance(lj, orientation) / scale; + double vDist = li.verticalDistance(lj, orientation) / scale; + // Line over or above + if (minHorizontalDistance <= hDist && hDist <= maxHorizontalDistance && minVerticalDistance <= vDist && vDist <= maxVerticalDistance) { + sets.union(li, lj); + } + // Split line that needs later merging + else if (minHorizontalMergeDistance <= hDist && hDist <= maxHorizontalMergeDistance && minVerticalMergeDistance <= vDist && vDist <= maxVerticalMergeDistance) { + sets.union(li, lj); + } + } + } + } + List> zones = new ArrayList>(); + for (Set group : sets) { + zones.add(new ArrayList(group)); + } + return zones; + } + + + private List> mergeZones(List> zones, double tolerance) { + + List bounds = new ArrayList(zones.size()); + for (List zone : zones) { + BoundingBoxBuilder builder = new BoundingBoxBuilder(); + for (ComponentLine line : zone) { + for (Character component : line.getComponents()) { + builder.expand(component.getChunk()); + } + } + bounds.add(builder.getBounds()); + } + + List> outputZones = new ArrayList>(); + mainFor: + for (int i = 0; i < zones.size(); i++) { + for (int j = 0; j < zones.size(); j++) { + if (i == j || bounds.get(j) == null || bounds.get(i) == null) { + continue; + } + if (bounds.get(j).contains(bounds.get(i), tolerance)) { + zones.get(j).addAll(zones.get(i)); + bounds.set(i, null); + continue mainFor; + } + } + outputZones.add(zones.get(i)); + } + return outputZones; + } + + + private List> mergeLines(List> zones, + double orientation, + double minHorizontalDistance, + double maxHorizontalDistance, + double minVerticalDistance, + double maxVerticalDistance) { + + List> outputZones = new ArrayList>(zones.size()); + for (List zone : zones) { + outputZones.add(mergeLinesInZone(zone, orientation, minHorizontalDistance, maxHorizontalDistance, minVerticalDistance, maxVerticalDistance)); + } + return outputZones; + } + + + private List mergeLinesInZone(List lines, + double orientation, + double minHorizontalDistance, + double maxHorizontalDistance, + double minVerticalDistance, + double maxVerticalDistance) { + + DisjointSets sets = new DisjointSets(lines); + for (int i = 0; i < lines.size(); i++) { + ComponentLine li = lines.get(i); + for (int j = i + 1; j < lines.size(); j++) { + ComponentLine lj = lines.get(j); + double hDist = li.horizontalDistance(lj, orientation); + double vDist = li.verticalDistance(lj, orientation); + if (minHorizontalDistance <= hDist && hDist <= maxHorizontalDistance && minVerticalDistance <= vDist && vDist <= maxVerticalDistance) { + sets.union(li, lj); + } else if (minVerticalDistance <= vDist && vDist <= maxVerticalDistance && Math.abs(hDist - Math.min(li.getLength(), lj.getLength())) < 0.1) { + boolean componentOverlap = false; + int overlappingCount = 0; + for (Character ci : li.getComponents()) { + for (Character cj : lj.getComponents()) { + double dist = ci.overlappingDistance(cj, orientation); + if (dist > 2) { + componentOverlap = true; + } + if (dist > 0) { + overlappingCount++; + } + } + } + if (!componentOverlap && overlappingCount <= 2) { + sets.union(li, lj); + } + } + } + } + List outputZone = new ArrayList(); + for (Set group : sets) { + List components = new ArrayList(); + for (ComponentLine line : group) { + components.addAll(line.getComponents()); + } + Collections.sort(components, Character.CharacterXComparator.getInstance()); + outputZone.add(new ComponentLine(components, orientation)); + } + return outputZone; + } + + + /** + * Converts list of zones from internal format (using components and + * component lines) to BxModel. + * + * @param zones zones in internal format + * @param wordSpacing - maximum allowed distance between components that + * belong to one word + * @return BxModel page + */ + private List convertToBxModel(List> zones, double wordSpacing) { + + List zoneList = new ArrayList<>(); + if (zones.size() > MAX_ZONES_PER_PAGE) { + List oneZone = new ArrayList(); + for (List zone : zones) { + oneZone.addAll(zone); + } + zones = new ArrayList<>(); + zones.add(oneZone); + } + + for (List lines : zones) { + Zone zone = new Zone(); + for (ComponentLine line : lines) { + zone.addLine(line.convertToBxLine(wordSpacing)); + } + List zLines = Lists.newArrayList(zone.getLines()); + Collections.sort(zLines, new Comparator() { + + @Override + public int compare(Line o1, Line o2) { + + return Double.compare(o1.getbBox().getY(), o2.getbBox().getY()); + } + + }); + zone.setLines(zLines); + BoundingBoxBuilder.setBounds(zone); + zoneList.add(zone); + } + ZoneUtils.sortZonesYX(zoneList); + return zoneList; + } + + + /** + * Neighbor distance comparator based on the distance. + *

+ * The ordering is not consistent with equals. + */ + protected static final class NeighborDistanceComparator implements Comparator { + + private NeighborDistanceComparator() { + + } + + + @Override + public int compare(Neighbor o1, Neighbor o2) { + + return Double.compare(o1.getDistance(), o2.getDistance()); + } + + + private static final NeighborDistanceComparator instance = new NeighborDistanceComparator(); + + + public static NeighborDistanceComparator getInstance() { + + return instance; + } + + } + + /** + * Internal representation of the text line. + */ + protected static class ComponentLine { + + private final double x0; + private final double y0; + + private final double x1; + private final double y1; + + private final double height; + + private final List components; + + + public ComponentLine(List components, double orientation) { + + this.components = components; + + if (components.size() >= 2) { + // Simple linear regression + double sx = 0.0, sxx = 0.0, sxy = 0.0, sy = 0.0; + for (Character component : components) { + sx += component.getX(); + sxx += component.getX() * component.getX(); + sxy += component.getX() * component.getY(); + sy += component.getY(); + } + double b = (components.size() * sxy - sx * sy) / (components.size() * sxx - sx * sx); + double a = (sy - b * sx) / components.size(); + + this.x0 = components.get(0).getX(); + this.y0 = a + b * this.x0; + this.x1 = components.get(components.size() - 1).getX(); + this.y1 = a + b * this.x1; + } else if (!components.isEmpty()) { + Character component = components.get(0); + double dx = component.getChunk().getWidthDirAdj() / 3; + double dy = dx * Math.tan(orientation); + this.x0 = component.getX() - dx; + this.x1 = component.getX() + dx; + this.y0 = component.getY() - dy; + this.y1 = component.getY() + dy; + } else { + throw new IllegalArgumentException("Component list must not be empty"); + } + height = computeHeight(); + } + + + public double getAngle() { + + return Math.atan2(y1 - y0, x1 - x0); + } + + + public double getSlope() { + + return (y1 - y0) / (x1 - x0); + } + + + public double getLength() { + + return Math.sqrt((x0 - x1) * (x0 - x1) + (y0 - y1) * (y0 - y1)); + } + + + private double computeHeight() { + + double sum = 0.0; + for (Character component : components) { + sum += component.getHeight(); + } + return sum / components.size(); + } + + + public double getHeight() { + + return height; + } + + + public List getComponents() { + + return components; + } + + + public double angularDifference(ComponentLine j) { + + double diff = Math.abs(getAngle() - j.getAngle()); + if (diff <= Math.PI / 2) { + return diff; + } else { + return Math.PI - diff; + } + } + + + public double horizontalDistance(ComponentLine other, double orientation) { + + double[] xs = new double[4]; + double s = Math.sin(-orientation), c = Math.cos(-orientation); + xs[0] = c * x0 - s * y0; + xs[1] = c * x1 - s * y1; + xs[2] = c * other.x0 - s * other.y0; + xs[3] = c * other.x1 - s * other.y1; + boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0]; + Arrays.sort(xs); + return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1); + } + + + public double verticalDistance(ComponentLine other, double orientation) { + + double xm = (x0 + x1) / 2, ym = (y0 + y1) / 2; + double xn = (other.x0 + other.x1) / 2, yn = (other.y0 + other.y1) / 2; + double a = Math.tan(orientation); + return Math.abs(a * (xn - xm) + ym - yn) / Math.sqrt(a * a + 1); + } + + + public Line convertToBxLine(double wordSpacing) { + + Line line = new Line(); + Word word = new Word(); + Character previousComponent = null; + for (Character component : components) { + if (previousComponent != null) { + double dist = component.getChunk().getXDirAdj() - previousComponent.getChunk().getXDirAdj() - previousComponent.getChunk().getWidthDirAdj(); + if (dist > wordSpacing) { + BoundingBoxBuilder.setBounds(word); + line.addWord(word); + word = new Word(); + } + } + word.addChunk(component.getChunk()); + previousComponent = component; + } + BoundingBoxBuilder.setBounds(word); + line.addWord(word); + BoundingBoxBuilder.setBounds(line); + return line; + } + + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/HierarchicalReadingOrderResolver.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/HierarchicalReadingOrderResolver.java new file mode 100644 index 0000000..ff5bc14 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/HierarchicalReadingOrderResolver.java @@ -0,0 +1,272 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.google.common.collect.Lists; +import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.BBoxObject; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Line; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Word; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Zone; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.utils.DoubleUtils; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.BBoxZoneGroup; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.DistElem; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.DocumentPlane; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder.TreeToListConverter; + +@Service +public class HierarchicalReadingOrderResolver { + + static final int GRIDSIZE = 50; + static final double BOXES_FLOW = 0.5; + static final double EPS = 0.01; + static final int MAX_ZONES = 1000; + static final Comparator Y_ASCENDING_ORDER = new Comparator() { + + @Override + public int compare(BBoxObject o1, BBoxObject o2) { + + return DoubleUtils.compareDouble(o1.getY(), o2.getY(), EPS); + } + }; + + static final Comparator X_ASCENDING_ORDER = new Comparator() { + + @Override + public int compare(BBoxObject o1, BBoxObject o2) { + + return DoubleUtils.compareDouble(o1.getX(), o2.getX(), EPS); + } + }; + + static final Comparator TP_X_ASCENDING_ORDER = new Comparator() { + + @Override + public int compare(RedTextPosition o1, RedTextPosition o2) { + + return DoubleUtils.compareDouble(o1.getXDirAdj(), o2.getXDirAdj(), EPS); + } + }; + + static final Comparator YX_ASCENDING_ORDER = new Comparator() { + + @Override + public int compare(BBoxObject o1, BBoxObject o2) { + + int yCompare = Y_ASCENDING_ORDER.compare(o1, o2); + return yCompare == 0 ? X_ASCENDING_ORDER.compare(o1, o2) : yCompare; + } + }; + + + public List resolve(List zones) { + + for (Zone zone : zones) { + List lines = Lists.newArrayList(zone.getLines()); + for (Line line : lines) { + List words = Lists.newArrayList(line.getWords()); + for (Word word : words) { + List chunks = Lists.newArrayList(word.getTextPositions()); + Collections.sort(chunks, TP_X_ASCENDING_ORDER); + word.setTextPositions(chunks); + } + Collections.sort(words, X_ASCENDING_ORDER); + line.setWords(words); + } + Collections.sort(lines, YX_ASCENDING_ORDER); + zone.setLines(lines); + } + List orderedZones; + if (zones.size() > MAX_ZONES) { + orderedZones = new ArrayList(zones); + Collections.sort(orderedZones, YX_ASCENDING_ORDER); + } else { + orderedZones = reorderZones(zones); + } + return orderedZones; + } + + + private List reorderZones(List unorderedZones) { + + if (unorderedZones.isEmpty()) { + return new ArrayList(); + } else if (unorderedZones.size() == 1) { + List ret = new ArrayList(1); + ret.add(unorderedZones.get(0)); + return ret; + } else { + BBoxZoneGroup bxZonesTree = groupZonesHierarchically(unorderedZones); + sortGroupedZones(bxZonesTree); + TreeToListConverter treeConverter = new TreeToListConverter(); + List orderedZones = treeConverter.convertToList(bxZonesTree); + assert unorderedZones.size() == orderedZones.size(); + return orderedZones; + } + } + + + /** + * Builds a binary tree of zones and groups of zones from a list of unordered zones. This is done in hierarchical + * clustering by joining two least distant nodes. Distance is calculated in the distance() method. + * + * @param zones is a list of unordered zones + * @return root of the zones clustered in a tree + */ + private BBoxZoneGroup groupZonesHierarchically(List zones) { + /* + * Distance tuples are stored sorted by ascending distance value + */ + List> dists = new ArrayList>(zones.size() * zones.size() / 2); + for (int idx1 = 0; idx1 < zones.size(); ++idx1) { + for (int idx2 = idx1 + 1; idx2 < zones.size(); ++idx2) { + Zone zone1 = zones.get(idx1); + Zone zone2 = zones.get(idx2); + dists.add(new DistElem(false, distance(zone1, zone2), zone1, zone2)); + } + } + Collections.sort(dists); + DocumentPlane plane = new DocumentPlane(zones, GRIDSIZE); + while (!dists.isEmpty()) { + DistElem distElem = dists.get(0); + dists.remove(0); + if (!distElem.isC() && plane.anyObjectsBetween(distElem.getObj1(), distElem.getObj2())) { + dists.add(new DistElem(true, distElem.getDist(), distElem.getObj1(), distElem.getObj2())); + continue; + } + BBoxZoneGroup newGroup = new BBoxZoneGroup(distElem.getObj1(), distElem.getObj2()); + plane.remove(distElem.getObj1()).remove(distElem.getObj2()); + dists = removeDistElementsContainingObject(dists, distElem.getObj1()); + dists = removeDistElementsContainingObject(dists, distElem.getObj2()); + for (BBoxObject other : plane.getObjects()) { + dists.add(new DistElem(false, distance(other, newGroup), newGroup, other)); + } + Collections.sort(dists); + plane.add(newGroup); + } + + assert plane.getObjects().size() == 1 : "There should be one object left at the plane after grouping"; + return (BBoxZoneGroup) plane.getObjects().get(0); + } + + + /** + * Removes all distance tuples containing obj + */ + private List> removeDistElementsContainingObject(Collection> list, BBoxObject obj) { + + List> ret = new ArrayList>(); + for (DistElem distElem : list) { + if (distElem.getObj1() != obj && distElem.getObj2() != obj) { + ret.add(distElem); + } + } + return ret; + } + + + /** + * Swaps children of BxZoneGroup if necessary. A group with smaller sort factor is placed to the left (leftChild). + * An object with greater sort factor is placed on the right (rightChild). This plays an important role when + * traversing the tree in conversion to a one dimensional list. + * + * @param group + */ + private void sortGroupedZones(BBoxZoneGroup group) { + + BBoxObject leftChild = group.getLeftChild(); + BBoxObject rightChild = group.getRightChild(); + if (shouldBeSwapped(leftChild, rightChild)) { + // swap + group.setLeftChild(rightChild); + group.setRightChild(leftChild); + } + + if (leftChild instanceof BBoxZoneGroup) // if the child is a tree node, then recurse + { + sortGroupedZones((BBoxZoneGroup) leftChild); + } + if (rightChild instanceof BBoxZoneGroup) // as above - recurse + { + sortGroupedZones((BBoxZoneGroup) rightChild); + } + } + + + private boolean shouldBeSwapped(BBoxObject first, BBoxObject second) { + + double cx, cy, cw, ch, ox, oy, ow, oh; + cx = first.getbBox().getX(); + cy = first.getbBox().getY(); + cw = first.getbBox().getWidth(); + ch = first.getbBox().getHeight(); + + ox = second.getbBox().getX(); + oy = second.getbBox().getY(); + ow = second.getbBox().getWidth(); + oh = second.getbBox().getHeight(); + + // Determine Octant + // + // 0 | 1 | 2 + // __|___|__ + // 7 | 9 | 3 First is placed in 9th square + // __|___|__ + // 6 | 5 | 4 + + if (cx + cw <= ox) { //2,3,4 + return false; + } else if (ox + ow <= cx) { //0,6,7 + return true; //6 + } else if (cy + ch <= oy) { + return false; //5 + } else if (oy + oh <= cy) { + return true; //1 + } else { //two zones + double xdiff = ox + ow / 2 - cx - cw / 2; + double ydiff = oy + oh / 2 - cy - ch / 2; + return xdiff + ydiff < 0; + } + } + + + /** + * A distance function between two TextBoxes. + *

+ * Consider the bounding rectangle for obj1 and obj2. Return its area minus the areas of obj1 and obj2, shown as + * 'www' below. This value may be negative. (x0,y0) +------+..........+ | obj1 |wwwwwwwwww: +------+www+------+ + * :wwwwwwwwww| obj2 | +..........+------+ (x1,y1) + * + * @return distance value based on objects' coordinates and physical size on a plane + */ + private double distance(BBoxObject obj1, BBoxObject obj2) { + + double x0 = Math.min(obj1.getX(), obj2.getX()); + double y0 = Math.min(obj1.getY(), obj2.getY()); + double x1 = Math.max(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth()); + double y1 = Math.max(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight()); + double dist = ((x1 - x0) * (y1 - y0) - obj1.getArea() - obj2.getArea()); + + double obj1X = obj1.getX(); + double obj1CenterX = obj1.getX() + obj1.getWidth() / 2; + double obj1CenterY = obj1.getY() + obj1.getHeight() / 2; + double obj2X = obj2.getX(); + double obj2CenterX = obj2.getX() + obj2.getWidth() / 2; + double obj2CenterY = obj2.getY() + obj2.getHeight() / 2; + + double obj1obj2VectorCosineAbsLeft = Math.abs((obj2X - obj1X) / Math.sqrt((obj2X - obj1X) * (obj2X - obj1X) + (obj2CenterY - obj1CenterY) * (obj2CenterY - obj1CenterY))); + double obj1obj2VectorCosineAbsCenter = Math.abs((obj2CenterX - obj1CenterX) / Math.sqrt((obj2CenterX - obj1CenterX) * (obj2CenterX - obj1CenterX) + (obj2CenterY - obj1CenterY) * (obj2CenterY - obj1CenterY))); + + double cosine = Math.min(obj1obj2VectorCosineAbsLeft, obj1obj2VectorCosineAbsCenter); + + final double MAGIC_COEFF = 0.5; + return dist * (MAGIC_COEFF + cosine); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/DisjointSets.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/DisjointSets.java new file mode 100644 index 0000000..e4cc563 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/DisjointSets.java @@ -0,0 +1,212 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model; + +import java.util.AbstractSet; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; + +public class DisjointSets implements Iterable> { + + private final Map> map = new HashMap>(); + + + /** + * Constructs a new set of singletons. + * + * @param c elements of singleton sets + */ + public DisjointSets(Collection c) { + + for (E element : c) { + map.put(element, new Entry(element)); + } + } + + + /** + * Checks if elements are in the same subsets. + * + * @param e1 element from a subset + * @param e2 element from a subset + * @return true if elements are in the same subset; false otherwise + */ + public boolean areTogether(E e1, E e2) { + + return map.get(e1).findRepresentative() == map.get(e2).findRepresentative(); + } + + + /** + * Merges subsets which elements e1 and e2 belong to. + * + * @param e1 element from a subset + * @param e2 element from a subset + */ + public void union(E e1, E e2) { + + Entry r1 = map.get(e1).findRepresentative(); + Entry r2 = map.get(e2).findRepresentative(); + if (r1 != r2) { + if (r1.size <= r2.size) { + r2.mergeWith(r1); + } else { + r1.mergeWith(r2); + } + } + } + + + @Override + public Iterator> iterator() { + + return new Iterator>() { + + private final Iterator> iterator = map.values().iterator(); + private Entry nextRepresentative; + + { + findNextRepresentative(); + } + + @Override + public boolean hasNext() { + + return nextRepresentative != null; + } + + + @Override + public Set next() { + + if (nextRepresentative == null) { + throw new NoSuchElementException(); + } + Set result = nextRepresentative.asSet(); + findNextRepresentative(); + return result; + } + + + private void findNextRepresentative() { + + while (iterator.hasNext()) { + Entry candidate = iterator.next(); + if (candidate.isRepresentative()) { + nextRepresentative = candidate; + return; + } + } + nextRepresentative = null; + } + + + @Override + public void remove() { + + throw new UnsupportedOperationException(); + } + + }; + } + + + private static class Entry { + + private int size = 1; + private final E value; + private Entry parent = this; + private Entry next = null; + private Entry last = this; + + + Entry(E value) { + + this.value = value; + } + + + void mergeWith(Entry otherRepresentative) { + + size += otherRepresentative.size; + last.next = otherRepresentative; + last = otherRepresentative.last; + otherRepresentative.parent = this; + } + + + Entry findRepresentative() { + + Entry representative = parent; + while (representative.parent != representative) { + representative = representative.parent; + } + for (Entry entry = this; entry != representative; ) { + Entry nextEntry = entry.parent; + entry.parent = representative; + entry = nextEntry; + } + return representative; + } + + + boolean isRepresentative() { + + return parent == this; + } + + + Set asSet() { + + return new AbstractSet() { + + @Override + public Iterator iterator() { + + return new Iterator() { + + private Entry nextEntry = findRepresentative(); + + + @Override + public boolean hasNext() { + + return nextEntry != null; + } + + + @Override + public E next() { + + if (nextEntry == null) { + throw new NoSuchElementException(); + } + E result = nextEntry.value; + nextEntry = nextEntry.next; + return result; + } + + + @Override + public void remove() { + + throw new UnsupportedOperationException(); + } + + }; + } + + + @Override + public int size() { + + return findRepresentative().size; + } + }; + } + + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Histogram.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Histogram.java new file mode 100644 index 0000000..8af304e --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/Histogram.java @@ -0,0 +1,199 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class Histogram implements Iterable { + + private static final double EPSILON = 1.0e-6; + + private final double min; + private final double delta; + private final double resolution; + + private double[] frequencies; + + + /** + * Constructs a new histogram for values in range [minValue, maxValue] with + * given resolution. + * + * @param minValue - minimum allowed value + * @param maxValue - maximum allowed value + * @param resolution - histogram's resolution + */ + public Histogram(double minValue, double maxValue, double resolution) { + + this.min = minValue - EPSILON; + this.delta = maxValue - minValue + 2 * EPSILON; + int size = Math.max(1, (int) Math.round((maxValue - minValue) / resolution)); + this.resolution = this.delta / size; + this.frequencies = new double[size]; + } + + + public void kernelSmooth(double[] kernel) { + + double[] newFrequencies = new double[frequencies.length]; + int shift = (kernel.length - 1) / 2; + for (int i = 0; i < kernel.length; i++) { + int jStart = Math.max(0, i - shift); + int jEnd = Math.min(frequencies.length, frequencies.length + i - shift); + for (int j = jStart; j < jEnd; j++) { + newFrequencies[j - i + shift] += kernel[i] * frequencies[j]; + } + } + frequencies = newFrequencies; + } + + + public void circularKernelSmooth(double[] kernel) { + + double[] newFrequencies = new double[frequencies.length]; + int shift = (kernel.length - 1) / 2; + for (int i = 0; i < frequencies.length; i++) { + for (int d = 0; d < kernel.length; d++) { + int j = i + d - shift; + if (j < 0) { + j += frequencies.length; + } else if (j >= frequencies.length) { + j -= frequencies.length; + } + newFrequencies[i] += kernel[d] * frequencies[j]; + } + } + frequencies = newFrequencies; + } + + + public double[] createGaussianKernel(double length, double stdDeviation) { + + int r = (int) Math.round(length / resolution) / 2; + stdDeviation /= resolution; + + int size = 2 * r + 1; + double[] kernel = new double[size]; + double sum = 0; + double b = 2 * stdDeviation * stdDeviation; + double a = 1 / Math.sqrt(Math.PI * b); + for (int i = 0; i < size; i++) { + kernel[i] = a * Math.exp(-(i - r) * (i - r) / b); + sum += kernel[i]; + } + for (int i = 0; i < size; i++) { + kernel[i] /= sum; + } + return kernel; + } + + + public void circularGaussianSmooth(double windowLength, double stdDeviation) { + + circularKernelSmooth(createGaussianKernel(windowLength, stdDeviation)); + } + + + public void gaussianSmooth(double windowLength, double stdDeviation) { + + kernelSmooth(createGaussianKernel(windowLength, stdDeviation)); + } + + + /** + * Adds single occurrence of given value to the histogram. + * + * @param value inserted values + */ + public void add(double value) { + + frequencies[(int) ((value - min) / resolution)] += 1.0; + } + + + /** + * Returns histogram's number of bins. + * + * @return number of bins + */ + public int getSize() { + + return frequencies.length; + } + + + /** + * Finds the histogram's peak value. + * + * @return peak value + */ + public double getPeakValue() { + + int peakIndex = 0; + for (int i = 1; i < frequencies.length; i++) { + if (frequencies[i] > frequencies[peakIndex]) { + peakIndex = i; + } + } + int peakEndIndex = peakIndex + 1; + final double EPS = 0.0001; + while (peakEndIndex < frequencies.length && Math.abs(frequencies[peakEndIndex] - frequencies[peakIndex]) < EPS) { + peakEndIndex++; + } + return ((double) peakIndex + peakEndIndex) / 2 * resolution + min; + } + + + @Override + public Iterator iterator() { + + return new Iterator() { + + private int index = 0; + + + @Override + public boolean hasNext() { + + return index < frequencies.length; + } + + + @Override + public Object next() { + + if (index >= frequencies.length) { + throw new NoSuchElementException(); + } + return new Bin(index++); + } + + + @Override + public void remove() { + + throw new UnsupportedOperationException("Not supported yet."); + } + + }; + } + + + public final class Bin { + + private final int index; + + + private Bin(int index) { + + this.index = index; + } + + + public double getValue() { + + return (index + 0.5) * resolution + min; + } + + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/BBoxObject.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/BBoxObject.java new file mode 100644 index 0000000..958e4e4 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/BBoxObject.java @@ -0,0 +1,49 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor; + +public abstract class BBoxObject { + + private BoundingBox bBox; + + + public double getArea() { + + return (bBox.getHeight() * bBox.getWidth()); + } + + + public BoundingBox getbBox() { + + return bBox; + } + + + public void setbBox(BoundingBox bBox) { + + this.bBox = bBox; + } + + + public double getX() { + + return bBox.getX(); + } + + + public double getY() { + + return bBox.getY(); + } + + + public double getWidth() { + + return bBox.getWidth(); + } + + + public double getHeight() { + + return bBox.getHeight(); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/BoundingBox.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/BoundingBox.java new file mode 100644 index 0000000..81173b8 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/BoundingBox.java @@ -0,0 +1,21 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public final class BoundingBox { + + private final double x; + private final double y; + private final double width; + private final double height; + + + public boolean contains(BoundingBox contained, double tolerance) { + + return x <= contained.getX() + tolerance && y <= contained.getY() + tolerance && x + width >= contained.getX() + contained.getWidth() - tolerance && y + height >= contained.getY() + contained.getHeight() - tolerance; + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Line.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Line.java new file mode 100644 index 0000000..aa9dc9f --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Line.java @@ -0,0 +1,21 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Getter; +import lombok.Setter; + +public class Line extends BBoxObject { + + @Setter + @Getter + private List words = new ArrayList<>(); + + + public void addWord(Word word) { + + this.words.add(word); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Word.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Word.java new file mode 100644 index 0000000..4b1fbdc --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Word.java @@ -0,0 +1,23 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor; + +import java.util.ArrayList; +import java.util.List; + +import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition; + +import lombok.Getter; +import lombok.Setter; + +public class Word extends BBoxObject { + + @Setter + @Getter + private List textPositions = new ArrayList<>(); + + + public void addChunk(RedTextPosition chunk) { + + this.textPositions.add(chunk); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Zone.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Zone.java new file mode 100644 index 0000000..5f3eb1d --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/Zone.java @@ -0,0 +1,19 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Data; + +@Data +public final class Zone extends BBoxObject { + + private List lines = new ArrayList<>(); + + + public void addLine(Line line) { + + this.lines.add(line); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/docstrum/Character.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/docstrum/Character.java new file mode 100644 index 0000000..e34e6cc --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/docstrum/Character.java @@ -0,0 +1,110 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.docstrum; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition; + +import lombok.Data; +import lombok.Getter; + +@Data +public class Character { + + private final double x; + private final double y; + private final RedTextPosition chunk; + + private List neighbors; + + + public Character(RedTextPosition chunk) { + + this.x = chunk.getXDirAdj() + chunk.getWidthDirAdj() / 2; + this.y = chunk.getYDirAdj() + chunk.getHeightDir() / 2; + this.chunk = chunk; + } + + + public double getHeight() { + + return chunk.getHeightDir(); + } + + + public double distance(Character character) { + + double dx = getX() - character.getX(); + double dy = getY() - character.getY(); + return Math.sqrt(dx * dx + dy * dy); + } + + + public double horizontalDistance(Character character) { + + return Math.abs(getX() - character.getX()); + } + + + public double verticalDistance(Character character) { + + return Math.abs(getY() - character.getY()); + } + + + public void setNeighbors(List neighbors) { + + this.neighbors = neighbors; + } + + + public double angle(Character character) { + + if (getX() > character.getX()) { + return Math.atan2(getY() - character.getY(), getX() - character.getX()); + } else { + return Math.atan2(character.getY() - getY(), character.getX() - getX()); + } + } + + + public double overlappingDistance(Character other, double orientation) { + + double[] xs = new double[4]; + double s = Math.sin(-orientation), c = Math.cos(-orientation); + xs[0] = c * x - s * y; + xs[1] = c * (x + chunk.getWidthDirAdj()) - s * (y + chunk.getHeightDir()); + xs[2] = c * other.x - s * other.y; + xs[3] = c * (other.x + other.chunk.getWidthDirAdj()) - s * (other.y + other.chunk.getHeightDir()); + boolean overlapping = xs[1] >= xs[2] && xs[3] >= xs[0]; + Arrays.sort(xs); + return Math.abs(xs[2] - xs[1]) * (overlapping ? 1 : -1); + } + + + /** + * Component comparator based on x coordinate of the centroid of component. + *

+ * The ordering is not consistent with equals. + */ + public static final class CharacterXComparator implements Comparator { + + private CharacterXComparator() { + + } + + + @Override + public int compare(Character o1, Character o2) { + + return Double.compare(o1.getX(), o2.getX()); + } + + + @Getter + private static final CharacterXComparator instance = new CharacterXComparator(); + + } + +} \ No newline at end of file diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/docstrum/Neighbor.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/docstrum/Neighbor.java new file mode 100644 index 0000000..856f514 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/docstrum/Neighbor.java @@ -0,0 +1,36 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.docstrum; + +import lombok.Getter; + +public class Neighbor { + + @Getter + private final double distance; + @Getter + private final double angle; + private final Character originCharacter; + @Getter + private final Character character; + + + public Neighbor(Character neighbor, Character origin) { + + this.distance = neighbor.distance(origin); + this.angle = neighbor.angle(origin); + this.character = neighbor; + this.originCharacter = origin; + } + + + public double getHorizontalDistance() { + + return character.horizontalDistance(originCharacter); + } + + + public double getVerticalDistance() { + + return character.verticalDistance(originCharacter); + } + +} \ No newline at end of file diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/BoundingBoxBuilder.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/BoundingBoxBuilder.java new file mode 100644 index 0000000..d77ac03 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/BoundingBoxBuilder.java @@ -0,0 +1,96 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.utils; + +import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.BoundingBox; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Line; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Word; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Zone; + +public class BoundingBoxBuilder { + + private double minX = Double.POSITIVE_INFINITY; + private double minY = Double.POSITIVE_INFINITY; + private double maxX = Double.NEGATIVE_INFINITY; + private double maxY = Double.NEGATIVE_INFINITY; + + + public void expandByLines(Zone zone) { + + for (Line line : zone.getLines()) { + expand(line.getbBox()); + } + } + + + public void expandByWords(Line line) { + + for (Word word : line.getWords()) { + expand(word.getbBox()); + } + } + + + public void expandByChunks(Word word) { + + for (RedTextPosition chunk : word.getTextPositions()) { + expand(chunk); + } + } + + + public void expand(BoundingBox bounds) { + + if (bounds != null) { + minX = Math.min(minX, bounds.getX()); + minY = Math.min(minY, bounds.getY()); + maxX = Math.max(maxX, bounds.getX() + bounds.getWidth()); + maxY = Math.max(maxY, bounds.getY() + bounds.getHeight()); + } + } + + + public void expand(RedTextPosition bounds) { + + if (bounds != null) { + minX = Math.min(minX, bounds.getXDirAdj()); + minY = Math.min(minY, bounds.getYDirAdj()); + maxX = Math.max(maxX, bounds.getXDirAdj() + bounds.getWidthDirAdj()); + maxY = Math.max(maxY, bounds.getYDirAdj() + bounds.getHeightDir()); + } + } + + + public BoundingBox getBounds() { + + if (minX <= maxX && minY <= maxY) { + return new BoundingBox(minX, minY, maxX - minX, maxY - minY); + } else { + return new BoundingBox(0, 0, 0, 0); + } + } + + + public static void setBounds(Zone zone) { + + BoundingBoxBuilder builder = new BoundingBoxBuilder(); + builder.expandByLines(zone); + zone.setbBox(builder.getBounds()); + } + + + public static void setBounds(Line line) { + + BoundingBoxBuilder builder = new BoundingBoxBuilder(); + builder.expandByWords(line); + line.setbBox(builder.getBounds()); + } + + + public static void setBounds(Word word) { + + BoundingBoxBuilder builder = new BoundingBoxBuilder(); + builder.expandByChunks(word); + word.setbBox(builder.getBounds()); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/DoubleUtils.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/DoubleUtils.java new file mode 100644 index 0000000..2454536 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/DoubleUtils.java @@ -0,0 +1,18 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.utils; + +public class DoubleUtils { + + public static int compareDouble(double d1, double d2, double precision) { + + if (Double.isNaN(d1) || Double.isNaN(d2)) { + return Double.compare(d1, d2); + } + if (precision == 0) { + precision = 1; + } + long i1 = Math.round(d1 / precision); + long i2 = Math.round(d2 / precision); + return Long.valueOf(i1).compareTo(i2); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/ZoneUtils.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/ZoneUtils.java new file mode 100644 index 0000000..1a93cac --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/model/refactor/utils/ZoneUtils.java @@ -0,0 +1,36 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.utils; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Zone; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ZoneUtils { + + public void sortZonesYX(List zones) { + + sortZonesYX(zones, 0); + } + + + public void sortZonesYX(List zones, final double tolerance) { + + Collections.sort(zones, new Comparator() { + + @Override + public int compare(Zone o1, Zone o2) { + + int cmp = DoubleUtils.compareDouble(o1.getbBox().getY(), o2.getbBox().getY(), tolerance); + if (cmp == 0) { + return DoubleUtils.compareDouble(o1.getbBox().getX(), o2.getbBox().getX(), tolerance); + } + return cmp; + } + }); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/BBoxZoneGroup.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/BBoxZoneGroup.java new file mode 100644 index 0000000..061c85d --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/BBoxZoneGroup.java @@ -0,0 +1,63 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder; + +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.BBoxObject; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.BoundingBox; + +public class BBoxZoneGroup extends BBoxObject { + + private BBoxObject leftChild; + private BBoxObject rightChild; + + + public BBoxZoneGroup(BBoxObject child1, BBoxObject child2) { + + this.leftChild = child1; + this.rightChild = child2; + setBounds(Math.min(child1.getX(), child2.getX()), + Math.min(child1.getY(), child2.getY()), + Math.max(child1.getX() + child1.getWidth(), child2.getX() + child2.getWidth()), + Math.max(child1.getY() + child1.getHeight(), child2.getY() + child2.getHeight())); + } + + + public void setbBox(BoundingBox bBox) { + + super.setbBox(bBox); + } + + + public BBoxObject getLeftChild() { + + return leftChild; + } + + + public BBoxObject getRightChild() { + + return rightChild; + } + + + public BBoxZoneGroup setLeftChild(BBoxObject obj) { + + this.leftChild = obj; + return this; + } + + + public BBoxZoneGroup setRightChild(BBoxObject obj) { + + this.rightChild = obj; + return this; + } + + + public BBoxZoneGroup setBounds(double x0, double y0, double x1, double y1) { + + assert x1 >= x0; + assert y1 >= y0; + this.setbBox(new BoundingBox(x0, y0, x1 - x0, y1 - y0)); + return this; + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/DistElem.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/DistElem.java new file mode 100644 index 0000000..7755d8d --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/DistElem.java @@ -0,0 +1,115 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder; + +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.utils.DoubleUtils; + +public class DistElem implements Comparable> { + + @Override + public int hashCode() { + + final int prime = 31; + int result = 1; + result = prime * result + (c ? 1231 : 1237); + long temp; + temp = Double.doubleToLongBits(dist); + result = prime * result + (int) (temp ^ (temp >>> 32)); + result = prime * result + ((obj1 == null) ? 0 : obj1.hashCode()); + result = prime * result + ((obj2 == null) ? 0 : obj2.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + DistElem other = (DistElem) obj; + if (c != other.c) { + return false; + } + if (Double.doubleToLongBits(dist) != Double.doubleToLongBits(other.dist)) { + return false; + } + if (obj1 == null) { + if (other.obj1 != null) { + return false; + } + } else if (!obj1.equals(other.obj1)) { + return false; + } + if (obj2 == null) { + if (other.obj2 != null) { + return false; + } + } else if (!obj2.equals(other.obj2)) { + return false; + } + return true; + } + + + boolean c; + double dist; + E obj1; + E obj2; + + + public boolean isC() { + + return c; + } + + + public void setC(boolean c) { + + this.c = c; + } + + + public double getDist() { + + return dist; + } + + + public E getObj1() { + + return obj1; + } + + + public E getObj2() { + + return obj2; + } + + + public DistElem(boolean c, double dist, E obj1, E obj2) { + + this.c = c; + this.dist = dist; + this.obj1 = obj1; + this.obj2 = obj2; + } + + + @Override + public int compareTo(DistElem compareObject) { + + double eps = 1E-3; + if (c == compareObject.c) { + return DoubleUtils.compareDouble(dist, compareObject.dist, eps); + } else { + return c ? -1 : 1; + } + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/DocumentPlane.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/DocumentPlane.java new file mode 100644 index 0000000..71699be --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/DocumentPlane.java @@ -0,0 +1,280 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.BBoxObject; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.BoundingBox; +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Zone; + +/** + * A set-like data structure for objects placed on a plane. Can efficiently find objects in a certain rectangular area. + * It maintains two parallel lists of objects, each of which is sorted by its x or y coordinate. + * + * @author Pawel Szostek + */ +public class DocumentPlane { + + /** + * List of objects on the plane. Stored in a random order + */ + private final List objs; + /** + * Size of a grid square. If gridSize=50, then the plane is divided into squares of size 50. Each square contains + * objects placed in a 50x50 area + */ + private final int gridSize; + /** + * Redundant dictionary of objects on the plane. Allows efficient 2D space search. Keys are X-Y coordinates of a + * grid square. Single object can be stored under several keys (depending on its physical size). Grid squares are + * lazy-initialized. + */ + private final Map> grid; + + /** + * Representation of XY coordinates + */ + private static class GridXY { + + public int x; + public int y; + + + public GridXY(int x, int y) { + + this.x = x; + this.y = y; + } + + + @Override + public int hashCode() { + + return x * y; + } + + + @Override + public boolean equals(Object obj) { + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + GridXY comparedObj = (GridXY) obj; + return x == comparedObj.x && y == comparedObj.y; + } + + + @Override + public String toString() { + + return "(" + x + "," + y + ")"; + } + + } + + + public List getObjects() { + + return objs; + } + + + public DocumentPlane(List objectList, int gridSize) { + + this.grid = new HashMap>(); + this.objs = new ArrayList(); + this.gridSize = gridSize; + for (Zone obj : objectList) { + add(obj); + } + } + + + /** + * Looks for objects placed between obj1 and obj2 excluding them + * + * @param obj1 object + * @param obj2 object + * @return object list + */ + public List findObjectsBetween(BBoxObject obj1, BBoxObject obj2) { + + double x0 = Math.min(obj1.getX(), obj2.getX()); + double y0 = Math.min(obj1.getY(), obj2.getY()); + double x1 = Math.max(obj1.getX() + obj1.getWidth(), obj2.getX() + obj2.getWidth()); + double y1 = Math.max(obj1.getY() + obj1.getHeight(), obj2.getY() + obj2.getHeight()); + assert x1 >= x0 && y1 >= y0; + BoundingBox searchBounds = new BoundingBox(x0, y0, x1 - x0, y1 - y0); + List objsBetween = find(searchBounds); + /* + * the rectangle area must contain at least obj1 and obj2 + */ + objsBetween.remove(obj1); + objsBetween.remove(obj2); + return objsBetween; + } + + + /** + * Checks if there is any object placed between obj1 and obj2 + * + * @param obj1 object + * @param obj2 object + * @return true if anything is placed between, false otherwise + */ + public boolean anyObjectsBetween(BBoxObject obj1, BBoxObject obj2) { + + List lObjs = findObjectsBetween(obj1, obj2); + return !(lObjs.isEmpty()); + } + + + /** + * Adds object to the plane + * + * @param obj object + * @return document plane + */ + public DocumentPlane add(BBoxObject obj) { + + int objsBefore = this.objs.size(); + /* + * iterate over grid squares + */ + for (int y = ((int) obj.getY()) / gridSize; y <= ((int) (obj.getY() + obj.getHeight() + gridSize - 1)) / gridSize; ++y) { + for (int x = ((int) obj.getX()) / gridSize; x <= ((int) (obj.getX() + obj.getWidth() + gridSize - 1)) / gridSize; ++x) { + GridXY xy = new GridXY(x, y); + if (!grid.keySet().contains(xy)) { + /* + * add the non-existing key + */ + grid.put(xy, new ArrayList()); + grid.get(xy).add(obj); + assert grid.get(xy).size() == 1; + } else { + grid.get(xy).add(obj); + } + } + } + objs.add(obj); + /* + * size of the object list should be incremented + */ + assert objsBefore + 1 == objs.size(); + /* + * object list must contain the same number of objects as object dictionary + */ + assert objs.size() == elementsInGrid(); + return this; + } + + + public DocumentPlane remove(BBoxObject obj) { + /* + * iterate over grid squares + */ + for (int y = ((int) obj.getY()) / gridSize; y <= ((int) (obj.getY() + obj.getHeight() + gridSize - 1)) / gridSize; ++y) { + for (int x = ((int) obj.getX()) / gridSize; x <= ((int) (obj.getX() + obj.getWidth() + gridSize - 1)) / gridSize; ++x) { + GridXY xy = new GridXY(x, y); + if (grid.get(xy).contains(obj)) { + grid.get(xy).remove(obj); + } + } + } + objs.remove(obj); + assert objs.size() == elementsInGrid(); + return this; + } + + + /** + * Find objects within search bounds + * + * @param searchBounds is a search rectangle + * @return list of objects in!side search rectangle + */ + public List find(BoundingBox searchBounds) { + + List done = new ArrayList(); //contains already considered objects (wrt. optimization) + List ret = new ArrayList(); + double x0 = searchBounds.getX(); + double y0 = searchBounds.getY(); + double y1 = searchBounds.getY() + searchBounds.getHeight(); + double x1 = searchBounds.getX() + searchBounds.getWidth(); + /* + * iterate over grid squares + */ + for (int y = (int) y0 / gridSize; y < ((int) (y1 + gridSize - 1)) / gridSize; ++y) { + for (int x = (int) x0 / gridSize; x < ((int) (x1 + gridSize - 1)) / gridSize; ++x) { + GridXY xy = new GridXY(x, y); + if (!grid.containsKey(xy)) { + continue; + } + for (BBoxObject obj : grid.get(xy)) { + if (done.contains(obj)) /* + * omit if already checked + */ { + continue; + } + /* + * add to the checked objects + */ + done.add(obj); + /* + * check if two objects overlap + */ + if (obj.getX() + obj.getWidth() <= x0 || x1 <= obj.getX() || obj.getY() + obj.getHeight() <= y0 || y1 <= obj.getY()) { + continue; + } + ret.add(obj); + } + } + } + return ret; + } + + + /** + * Count objects stored in objects dictionary + * + * @return number of elements + */ + protected int elementsInGrid() { + + List objs_ = new ArrayList(); + for (GridXY coord : grid.keySet()) { + for (BBoxObject obj : grid.get(coord)) { + if (!objs_.contains(obj)) { + objs_.add(obj); + } + } + } + return objs_.size(); + } + + + public String dump() { + + StringBuilder sb = new StringBuilder(); + for (GridXY iter : grid.keySet()) { + sb.append(iter.toString()).append(" ["); + for (BBoxObject obj : grid.get(iter)) { + if (obj instanceof BBoxZoneGroup) { + BBoxZoneGroup group = (BBoxZoneGroup) obj; + sb.append(group.getLeftChild()); + sb.append(group.getRightChild()); + } else if (obj instanceof Zone) { + Zone zone = (Zone) obj; + sb.append(zone); + } + sb.append("\n"); + } + sb.append("]\n"); + } + return sb.toString(); + } + +} diff --git a/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/TreeToListConverter.java b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/TreeToListConverter.java new file mode 100644 index 0000000..7719246 --- /dev/null +++ b/layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/docstrum/readingorder/TreeToListConverter.java @@ -0,0 +1,32 @@ +package com.knecon.fforesight.service.layoutparser.processor.services.docstrum.readingorder; + +import java.util.ArrayList; +import java.util.List; + +import com.knecon.fforesight.service.layoutparser.processor.services.docstrum.model.refactor.Zone; + +/** + * @author Pawel Szostek + */ +public class TreeToListConverter { + + public List convertToList(BBoxZoneGroup obj) { + + List ret = new ArrayList(); + if (obj.getLeftChild() instanceof Zone) { + Zone zone = (Zone) obj.getLeftChild(); + ret.add(zone); + } else { // obj.getLeftChild() instanceof BxZoneGroup + ret.addAll(convertToList((BBoxZoneGroup) obj.getLeftChild())); + } + + if (obj.getRightChild() instanceof Zone) { + Zone zone = (Zone) obj.getRightChild(); + ret.add(zone); + } else { // obj.getRightChild() instanceof BxZoneGroup + ret.addAll(convertToList((BBoxZoneGroup) obj.getRightChild())); + } + return ret; + } + +} diff --git a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java index 0751be3..747f7a9 100644 --- a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java +++ b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java @@ -25,7 +25,9 @@ public class LayoutparserEnd2EndTest extends AbstractTest { @SneakyThrows public void testLayoutParserEndToEnd() { - prepareStorage("files/bdr/Wie weiter bei Kristeneinrichtungen.pdf"); + String s = "（"; + String s1 = "）"; + prepareStorage("files/Minimal Examples/WrongOrderPage1.pdf"); LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(LayoutParsingType.REDACT_MANAGER); LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest); Arrays.stream(finishedEvent.message().split("\n")).forEach(log::info); diff --git a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java index 5c5eae9..3755838 100644 --- a/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java +++ b/layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java @@ -25,7 +25,7 @@ public class ViewerDocumentTest extends BuildDocumentTest { @SneakyThrows public void testViewerDocument() { - String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"; + String fileName = "files/new/agb1.pdf"; String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf"; var documentFile = new ClassPathResource(fileName).getFile(); @@ -35,9 +35,10 @@ public class ViewerDocumentTest extends BuildDocumentTest { Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER); long start = System.currentTimeMillis(); layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true); - System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000); + System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000); } + @Test @Disabled @SneakyThrows @@ -51,7 +52,11 @@ public class ViewerDocumentTest extends BuildDocumentTest { var tableResponse = mapper.readValue(new ClassPathResource(tableFileName).getInputStream(), TableServiceResponse.class); var documentFile = new ClassPathResource(fileName).getFile(); - var classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE, documentFile, new ImageServiceResponse(), tableResponse, Path.of(fileName).getFileName().toFile().toString()); + var classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE, + documentFile, + new ImageServiceResponse(), + tableResponse, + Path.of(fileName).getFileName().toFile().toString()); ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null); LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService); Document document = DocumentGraphFactory.buildDocumentGraph(classificationDocument); diff --git a/layoutparser-service/layoutparser-service-server/src/test/resources/files/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf b/layoutparser-service/layoutparser-service-server/src/test/resources/files/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf new file mode 100644 index 0000000..13d3112 Binary files /dev/null and b/layoutparser-service/layoutparser-service-server/src/test/resources/files/95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf differ diff --git a/layoutparser-service/layoutparser-service-server/src/test/resources/files/Minimal Examples/JapanWord1.pdf b/layoutparser-service/layoutparser-service-server/src/test/resources/files/Minimal Examples/JapanWord1.pdf new file mode 100644 index 0000000..1842bb5 Binary files /dev/null and b/layoutparser-service/layoutparser-service-server/src/test/resources/files/Minimal Examples/JapanWord1.pdf differ diff --git a/layoutparser-service/layoutparser-service-server/src/test/resources/files/Minimal Examples/WrongOrderPage1.pdf b/layoutparser-service/layoutparser-service-server/src/test/resources/files/Minimal Examples/WrongOrderPage1.pdf new file mode 100644 index 0000000..a41d9ad Binary files /dev/null and b/layoutparser-service/layoutparser-service-server/src/test/resources/files/Minimal Examples/WrongOrderPage1.pdf differ diff --git a/layoutparser-service/layoutparser-service-server/src/test/resources/files/Plenarprotokoll 20_24Seite6.pdf b/layoutparser-service/layoutparser-service-server/src/test/resources/files/Plenarprotokoll 20_24Seite6.pdf new file mode 100644 index 0000000..f02ef0d Binary files /dev/null and b/layoutparser-service/layoutparser-service-server/src/test/resources/files/Plenarprotokoll 20_24Seite6.pdf differ diff --git a/layoutparser-service/layoutparser-service-server/src/test/resources/files/RedactManager_Documentation-en 1.pdf b/layoutparser-service/layoutparser-service-server/src/test/resources/files/RedactManager_Documentation-en 1.pdf new file mode 100644 index 0000000..1444fca Binary files /dev/null and b/layoutparser-service/layoutparser-service-server/src/test/resources/files/RedactManager_Documentation-en 1.pdf differ diff --git a/layoutparser-service/layoutparser-service-server/src/test/resources/files/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf b/layoutparser-service/layoutparser-service-server/src/test/resources/files/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf new file mode 100644 index 0000000..a6c58ea Binary files /dev/null and b/layoutparser-service/layoutparser-service-server/src/test/resources/files/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf differ