RED-5381: Use new codestyle
This commit is contained in:
deiflaender
parent
b10958cb6a
commit
ad455455df
@ -1,17 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
public class Page {
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
@ -13,9 +16,6 @@ import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
@Data
|
||||
@ -51,15 +51,20 @@ public class TextBlock extends AbstractTextContainer {
|
||||
private String classification;
|
||||
|
||||
|
||||
private TextDirection getDir(){
|
||||
private TextDirection getDir() {
|
||||
|
||||
return sequences.get(0).getDir();
|
||||
}
|
||||
|
||||
private float getPageHeight(){
|
||||
|
||||
private float getPageHeight() {
|
||||
|
||||
return sequences.get(0).getPageHeight();
|
||||
}
|
||||
|
||||
private float getPageWidth(){
|
||||
|
||||
private float getPageWidth() {
|
||||
|
||||
return sequences.get(0).getPageWidth();
|
||||
}
|
||||
|
||||
@ -70,16 +75,13 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return minY;
|
||||
}
|
||||
else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - maxX;
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - maxX;
|
||||
|
||||
}
|
||||
else if (getDir().getDegrees() == 270) {
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
|
||||
return getPageWidth() - maxY ;
|
||||
}
|
||||
else {
|
||||
return getPageWidth() - maxY;
|
||||
} else {
|
||||
return minX;
|
||||
}
|
||||
}
|
||||
@ -91,44 +93,30 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return maxY;
|
||||
}
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - minX;
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageWidth() - minY;
|
||||
|
||||
else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - minX;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 270) {
|
||||
return getPageWidth() - minY ;
|
||||
|
||||
}
|
||||
|
||||
else {
|
||||
} else {
|
||||
return maxX;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getY1() {
|
||||
|
||||
if (getDir().getDegrees() == 90 ) {
|
||||
return minX;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 180) {
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return minX;
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return maxY;
|
||||
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 270) {
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageHeight() - maxX;
|
||||
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
return getPageHeight() - maxY;
|
||||
}
|
||||
}
|
||||
@ -139,27 +127,18 @@ public class TextBlock extends AbstractTextContainer {
|
||||
public float getY2() {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return maxX;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 180 ) {
|
||||
return maxX;
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
|
||||
return minY;
|
||||
}
|
||||
|
||||
else if (getDir().getDegrees() == 270 ) {
|
||||
return getPageHeight() - minX;
|
||||
}
|
||||
|
||||
else {
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageHeight() - minX;
|
||||
} else {
|
||||
return getPageHeight() - minY;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
|
||||
|
||||
this.minX = minX;
|
||||
|
||||
@ -2,8 +2,6 @@ package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import static java.util.stream.Collectors.toSet;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
@ -34,8 +32,7 @@ public class BlockificationService {
|
||||
static final float THRESHOLD = 1f;
|
||||
|
||||
|
||||
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines,
|
||||
List<Ruling> verticalRulingLines, int rotation, PDRectangle cropbox) {
|
||||
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, int rotation, PDRectangle cropbox) {
|
||||
|
||||
// sortRotatedSequences(textPositions);
|
||||
|
||||
@ -49,17 +46,42 @@ public class BlockificationService {
|
||||
Float splitX1 = null;
|
||||
for (TextPositionSequence word : textPositions) {
|
||||
|
||||
boolean lineSeparation = word.getY1() - maxY > word.getHeight() * 1.25;
|
||||
boolean lineSeparation = word.getY1() - maxY > word.getHeight() * 1.25;
|
||||
boolean startFromTop = prev != null && word.getY1() < prev.getY1() - prev.getTextHeight();
|
||||
boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
|
||||
boolean xIsBeforeFirstX = prev != null && word.getX1() < minX;
|
||||
boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
|
||||
boolean splittedByRuling =
|
||||
isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) ||
|
||||
isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight())
|
||||
boolean splittedByRuling = isSplittedByRuling(maxX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY1(),
|
||||
verticalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight()) || isSplittedByRuling(minX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY2(),
|
||||
horizontalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight())
|
||||
|
||||
|| isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight())
|
||||
|| isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight());
|
||||
|| isSplittedByRuling(maxX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY1(),
|
||||
horizontalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight()) || isSplittedByRuling(minX,
|
||||
minY,
|
||||
word.getX1(),
|
||||
word.getY2(),
|
||||
verticalRulingLines,
|
||||
word.getDir().getDegrees(),
|
||||
cropbox.getWidth(),
|
||||
cropbox.getHeight());
|
||||
boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || xIsBeforeFirstX || splitByX || splitByDir || splittedByRuling)) {
|
||||
@ -122,8 +144,7 @@ public class BlockificationService {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
|
||||
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft
|
||||
.getMinY()) {
|
||||
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
|
||||
previousLeft.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
@ -131,8 +152,7 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
|
||||
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight
|
||||
.getMinY()) {
|
||||
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
|
||||
previousRight.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
@ -151,10 +171,8 @@ public class BlockificationService {
|
||||
while (itty.hasNext()) {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
|
||||
.equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY()) || previous != null && previous
|
||||
.getOrientation()
|
||||
.equals(Orientation.LEFT) && block.getOrientation()
|
||||
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
|
||||
previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
|
||||
.equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
|
||||
previous.add(block);
|
||||
itty.remove();
|
||||
@ -193,12 +211,10 @@ public class BlockificationService {
|
||||
styleFrequencyCounter.add(wordBlock.getFontStyle());
|
||||
|
||||
if (textBlock == null) {
|
||||
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock
|
||||
.getRotation());
|
||||
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation());
|
||||
} else {
|
||||
TextBlock spatialEntity = textBlock.union(wordBlock);
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
|
||||
.getHeight());
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
|
||||
}
|
||||
}
|
||||
|
||||
@ -211,19 +227,21 @@ public class BlockificationService {
|
||||
textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest());
|
||||
}
|
||||
|
||||
if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences()
|
||||
.stream()
|
||||
.map(t -> round(t.getY1(), 3))
|
||||
.collect(toSet())
|
||||
.size() == 1) {
|
||||
if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getY1(), 3)).collect(toSet()).size() == 1) {
|
||||
textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getX1));
|
||||
}
|
||||
return textBlock;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1,
|
||||
List<Ruling> rulingLines, float rotation, float pageWidth, float pageHeight) {
|
||||
private boolean isSplittedByRuling(float previousX2,
|
||||
float previousY1,
|
||||
float currentX1,
|
||||
float currentY1,
|
||||
List<Ruling> rulingLines,
|
||||
float rotation,
|
||||
float pageWidth,
|
||||
float pageHeight) {
|
||||
|
||||
for (Ruling ruling : rulingLines) {
|
||||
var line = CoordSystemHelper.convertToDirAdj(ruling, rotation, pageWidth, pageHeight);
|
||||
@ -235,13 +253,7 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter,
|
||||
boolean landscape) {
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
|
||||
|
||||
float minX = 10000;
|
||||
float maxX = -100;
|
||||
@ -270,7 +282,7 @@ public class BlockificationService {
|
||||
if (documentFontSizeCounter.getMostPopular() != null) {
|
||||
if (textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
|
||||
|
||||
if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0){
|
||||
if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
|
||||
if (textBlock.getY1() < minX) {
|
||||
minX = textBlock.getY1();
|
||||
}
|
||||
@ -310,7 +322,7 @@ public class BlockificationService {
|
||||
continue;
|
||||
}
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0){
|
||||
if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
|
||||
if (textBlock.getY1() < minX) {
|
||||
minX = textBlock.getMinY();
|
||||
}
|
||||
@ -342,16 +354,13 @@ public class BlockificationService {
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println("Page: " + page.getPageNumber() + " Landscape: " + page.isLandscape()+ " MinX: " + minX + " MaxX: " + maxX + " MinY: " +minY + " MaxY: " + maxY);
|
||||
System.out.println("Page: " + page.getPageNumber() + " Landscape: " + page.isLandscape() + " MinX: " + minX + " MaxX: " + maxX + " MinY: " + minY + " MaxY: " + maxY);
|
||||
|
||||
}
|
||||
return new Rectangle(new Point(minX, minY), maxX - minX, maxY - minY, 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private double round(float value, int decimalPoints) {
|
||||
|
||||
var d = Math.pow(10, decimalPoints);
|
||||
|
||||
@ -1,5 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
@ -7,12 +12,9 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@ -34,24 +36,21 @@ public class ClassificationService {
|
||||
for (Page page : document.getPages()) {
|
||||
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
|
||||
if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getY(), page.getCropBox().getHeight() - btf.getTopLeft().getX() - btf.getWidth()),
|
||||
btf.getHeight(),
|
||||
btf.getWidth(),
|
||||
0);
|
||||
} else if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0);
|
||||
} else if (page.getRotation() == 180) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() - btf.getHeight()),
|
||||
btf.getWidth(),
|
||||
btf.getHeight(),
|
||||
0);
|
||||
}
|
||||
|
||||
if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getY(),page.getCropBox().getHeight() - btf.getTopLeft().getX() -btf.getWidth()), btf.getHeight(), btf.getWidth(), 0);
|
||||
}
|
||||
else if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
|
||||
btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0);
|
||||
}
|
||||
|
||||
else if(page.getRotation() == 180){
|
||||
btf = new Rectangle(new Point( btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() -btf.getHeight()), btf.getWidth(), btf.getHeight(), 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
page.setBodyTextFrame(btf);
|
||||
|
||||
|
||||
|
||||
page.setBodyTextFrame(btf);
|
||||
|
||||
classifyPage(btf, page, document, headlineFontSizes);
|
||||
}
|
||||
@ -68,10 +67,9 @@ public class ClassificationService {
|
||||
}
|
||||
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
|
||||
List<Float> headlineFontSizes) {
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() +" B " + textBlock.getSequences().get(0).getDir());
|
||||
System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() + " B " + textBlock.getSequences().get(0).getDir());
|
||||
|
||||
// if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
// textBlock.setClassification("Other");
|
||||
@ -91,19 +89,21 @@ public class ClassificationService {
|
||||
// .getMostPopular()))
|
||||
{
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame,
|
||||
textBlock) && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
|
||||
document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
|
||||
.size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
|
||||
.getCountPerValue()
|
||||
.containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
|
||||
.get(0)
|
||||
.getTextPositions()
|
||||
.get(0)
|
||||
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
@ -111,28 +111,25 @@ public class ClassificationService {
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
|
||||
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter()
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
|
||||
textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
|
||||
.get(0)
|
||||
.getTextPositions()
|
||||
.get(0)
|
||||
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter()
|
||||
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular()) {
|
||||
.equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("italic") && !document.getFontStyleCounter()
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
|
||||
@ -12,14 +12,15 @@ import lombok.experimental.UtilityClass;
|
||||
@UtilityClass
|
||||
public class CoordSystemHelper {
|
||||
|
||||
|
||||
public Rectangle convertToDirAdj(Rectangle rectangle, float rotation, float pageWidth, float pageHeight) {
|
||||
|
||||
var topLeft = convertPoint(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rotation, pageWidth, pageHeight);
|
||||
var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() , rotation, pageWidth, pageHeight);
|
||||
var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rotation, pageWidth, pageHeight);
|
||||
|
||||
|
||||
return new Rectangle(new Point((float)topLeft.getX(),(float) topLeft.getY()), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()), 0);
|
||||
return new Rectangle(new Point((float) topLeft.getX(), (float) topLeft.getY()),
|
||||
(float) (bottomRight.getX() - topLeft.getX()),
|
||||
(float) (bottomRight.getY() - topLeft.getY()),
|
||||
0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
@SuppressWarnings("all")
|
||||
public class PositionUtils {
|
||||
|
||||
|
||||
public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
|
||||
|
||||
//TODO Currently this is not working for rotated pages.
|
||||
@ -19,10 +19,9 @@ public class PositionUtils {
|
||||
|
||||
double threshold = textBlock.getMostPopularWordHeight() * 3;
|
||||
|
||||
if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() &&
|
||||
textBlock.getMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth() &&
|
||||
textBlock.getMinY() + threshold > btf.getTopLeft().getY() &&
|
||||
textBlock.getMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
|
||||
if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() && textBlock.getMaxX() - threshold < btf.getTopLeft()
|
||||
.getX() + btf.getWidth() && textBlock.getMinY() + threshold > btf.getTopLeft().getY() && textBlock.getMaxY() - threshold < btf.getTopLeft()
|
||||
.getY() + btf.getHeight()) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@ -41,7 +40,7 @@ public class PositionUtils {
|
||||
return true;
|
||||
}
|
||||
|
||||
if(rotation == 180 && textBlock.getY2() < btf.getTopLeft().getY()){
|
||||
if (rotation == 180 && textBlock.getY2() < btf.getTopLeft().getY()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -60,7 +59,6 @@ public class PositionUtils {
|
||||
|
||||
public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
|
||||
|
||||
|
||||
if (btf == null || textBlock == null) {
|
||||
return false;
|
||||
}
|
||||
@ -104,11 +102,14 @@ public class PositionUtils {
|
||||
|
||||
|
||||
public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) {
|
||||
|
||||
return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
|
||||
}
|
||||
|
||||
|
||||
public Float getApproxLineCount(TextBlock textBlock) {
|
||||
|
||||
return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -147,7 +147,6 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getRotationAdjustedY() {
|
||||
@ -160,8 +159,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getX1() {
|
||||
|
||||
|
||||
return textPositions.get(0).getXDirAdj();
|
||||
return textPositions.get(0).getXDirAdj();
|
||||
|
||||
}
|
||||
|
||||
@ -170,25 +168,16 @@ public class TextPositionSequence implements CharSequence {
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getX2() {
|
||||
|
||||
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getY1() {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
return textPositions.get(0).getYDirAdj() - getTextHeight();
|
||||
return textPositions.get(0).getYDirAdj() - getTextHeight();
|
||||
|
||||
}
|
||||
|
||||
@ -197,15 +186,11 @@ public class TextPositionSequence implements CharSequence {
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getY2() {
|
||||
|
||||
|
||||
|
||||
|
||||
return textPositions.get(0).getYDirAdj();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getTextHeight() {
|
||||
|
||||
@ -101,18 +101,20 @@ public class PdfSegmentationService {
|
||||
PDRectangle pdr = pdPage.getMediaBox();
|
||||
|
||||
int rotation = pdPage.getRotation();
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) ||
|
||||
pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
|
||||
|
||||
PDRectangle cropbox = pdPage.getCropBox();
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight());
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
|
||||
stripper.getRulings(),
|
||||
stripper.getMinCharWidth(),
|
||||
stripper.getMaxCharHeight());
|
||||
Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), rotation, cropbox);
|
||||
|
||||
|
||||
float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
|
||||
page.setCropBoxArea(cropboxArea);
|
||||
|
||||
System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" +cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox().getWidth() + " h:" +pdPage.getMediaBox().getHeight());
|
||||
System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" + cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox()
|
||||
.getWidth() + " h:" + pdPage.getMediaBox().getHeight());
|
||||
|
||||
page.setRotation(rotation);
|
||||
page.setLandscape(isLandscape);
|
||||
|
||||
@ -31,31 +31,41 @@ public abstract class AbstractTextContainer {
|
||||
@JsonIgnore
|
||||
private Orientation orientation = Orientation.NONE;
|
||||
|
||||
|
||||
public abstract String getText();
|
||||
|
||||
|
||||
public boolean containsBlock(TextBlock other) {
|
||||
|
||||
return this.minX <= other.getX1() && this.maxX >= other.getX2() && this.minY >= other.getY1() && this.maxY <= other.getY2();
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(AbstractTextContainer other) {
|
||||
|
||||
return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(Rectangle other) {
|
||||
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
|
||||
|
||||
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft()
|
||||
.getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getHeight() {
|
||||
|
||||
return maxY - minY;
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getWidth() {
|
||||
|
||||
return maxX - minX;
|
||||
}
|
||||
|
||||
|
||||
@ -1,15 +1,27 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
@Service
|
||||
public class TableExtractionService {
|
||||
|
||||
@ -1,5 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.visualization.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
@ -7,24 +17,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PdfVisualisationService {
|
||||
|
||||
|
||||
public void visualizeParagraphs(Document classifiedDoc, PDDocument document) throws IOException {
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
@ -66,7 +67,6 @@ public class PdfVisualisationService {
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
|
||||
|
||||
for (AbstractTextContainer textBlock : analyzedPage.getTextBlocks()) {
|
||||
if (textBlock == null) {
|
||||
continue;
|
||||
@ -84,7 +84,10 @@ public class PdfVisualisationService {
|
||||
//
|
||||
// contentStream.addRect(analyzedPage.getBodyTextFrame().getX1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), analyzedPage.getBodyTextFrame().getY1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), width, height);
|
||||
|
||||
contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), analyzedPage.getBodyTextFrame().getTopLeft().getY(), analyzedPage.getBodyTextFrame().getWidth(), analyzedPage.getBodyTextFrame().getHeight());
|
||||
contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(),
|
||||
analyzedPage.getBodyTextFrame().getTopLeft().getY(),
|
||||
analyzedPage.getBodyTextFrame().getWidth(),
|
||||
analyzedPage.getBodyTextFrame().getHeight());
|
||||
|
||||
contentStream.stroke();
|
||||
|
||||
@ -111,7 +114,6 @@ public class PdfVisualisationService {
|
||||
|
||||
contentStream.endText();
|
||||
|
||||
|
||||
// contentStream.setNonStrokingColor(Color.BLUE);
|
||||
// contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f);
|
||||
//
|
||||
@ -140,6 +142,7 @@ public class PdfVisualisationService {
|
||||
|
||||
|
||||
private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
|
||||
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
|
||||
@ -170,4 +173,5 @@ public class PdfVisualisationService {
|
||||
contentStream.endText();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user