Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-5381: Use new codestyle

Browse Source
This commit is contained in:
deiflaender 2022-10-19 17:16:23 +02:00
parent b10958cb6a
commit ad455455df
11 changed files with 206 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java
View File

@ -1,17 +1,18 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.classification.model;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import lombok.Data;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.pdfbox.pdmodel.common.PDRectangle; import org.apache.pdfbox.pdmodel.common.PDRectangle;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import lombok.Data;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data @Data
@RequiredArgsConstructor @RequiredArgsConstructor
public class Page { public class Page {

73
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java
View File

@ -1,5 +1,8 @@
package com.iqser.red.service.redaction.v1.server.classification.model; package com.iqser.red.service.redaction.v1.server.classification.model;
import java.util.ArrayList;
import java.util.List;
import com.dslplatform.json.CompiledJson; import com.dslplatform.json.CompiledJson;
import com.dslplatform.json.JsonAttribute; import com.dslplatform.json.JsonAttribute;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
@ -13,9 +16,6 @@ import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@AllArgsConstructor @AllArgsConstructor
@Builder @Builder
@Data @Data
@ -51,15 +51,20 @@ public class TextBlock extends AbstractTextContainer {
private String classification; private String classification;
private TextDirection getDir(){ private TextDirection getDir() {
return sequences.get(0).getDir(); return sequences.get(0).getDir();
} }
private float getPageHeight(){
private float getPageHeight() {
return sequences.get(0).getPageHeight(); return sequences.get(0).getPageHeight();
} }
private float getPageWidth(){
private float getPageWidth() {
return sequences.get(0).getPageWidth(); return sequences.get(0).getPageWidth();
} }
@ -70,16 +75,13 @@ public class TextBlock extends AbstractTextContainer {
if (getDir().getDegrees() == 90) { if (getDir().getDegrees() == 90) {
return minY; return minY;
} } else if (getDir().getDegrees() == 180) {
else if (getDir().getDegrees() == 180) {
return getPageWidth() - maxX; return getPageWidth() - maxX;
} } else if (getDir().getDegrees() == 270) {
else if (getDir().getDegrees() == 270) {
return getPageWidth() - maxY ; return getPageWidth() - maxY;
} } else {
else {
return minX; return minX;
} }
} }
@ -91,44 +93,30 @@ public class TextBlock extends AbstractTextContainer {
if (getDir().getDegrees() == 90) { if (getDir().getDegrees() == 90) {
return maxY; return maxY;
} } else if (getDir().getDegrees() == 180) {
else if (getDir().getDegrees() == 180) {
return getPageWidth() - minX; return getPageWidth() - minX;
} } else if (getDir().getDegrees() == 270) {
return getPageWidth() - minY;
else if (getDir().getDegrees() == 270) { } else {
return getPageWidth() - minY ;
}
else {
return maxX; return maxX;
} }
} }
@JsonIgnore @JsonIgnore
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getY1() { public float getY1() {
if (getDir().getDegrees() == 90 ) { if (getDir().getDegrees() == 90) {
return minX; return minX;
} } else if (getDir().getDegrees() == 180) {
else if (getDir().getDegrees() == 180) {
return maxY; return maxY;
} } else if (getDir().getDegrees() == 270) {
else if (getDir().getDegrees() == 270) {
return getPageHeight() - maxX; return getPageHeight() - maxX;
} } else {
else {
return getPageHeight() - maxY; return getPageHeight() - maxY;
} }
} }
@ -140,26 +128,17 @@ public class TextBlock extends AbstractTextContainer {
if (getDir().getDegrees() == 90) { if (getDir().getDegrees() == 90) {
return maxX; return maxX;
} } else if (getDir().getDegrees() == 180) {
else if (getDir().getDegrees() == 180 ) {
return minY; return minY;
} } else if (getDir().getDegrees() == 270) {
else if (getDir().getDegrees() == 270 ) {
return getPageHeight() - minX; return getPageHeight() - minX;
} } else {
else {
return getPageHeight() - minY; return getPageHeight() - minY;
} }
} }
public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) { public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
this.minX = minX; this.minX = minX;

91
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java
View File

@ -2,8 +2,6 @@ package com.iqser.red.service.redaction.v1.server.classification.service;
import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toSet;
import java.awt.geom.Line2D;
import java.awt.geom.Point2D;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Iterator; import java.util.Iterator;
@ -34,8 +32,7 @@ public class BlockificationService {
static final float THRESHOLD = 1f; static final float THRESHOLD = 1f;
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, int rotation, PDRectangle cropbox) {
List<Ruling> verticalRulingLines, int rotation, PDRectangle cropbox) {
// sortRotatedSequences(textPositions); // sortRotatedSequences(textPositions);
@ -54,12 +51,37 @@ public class BlockificationService {
boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1(); boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
boolean xIsBeforeFirstX = prev != null && word.getX1() < minX; boolean xIsBeforeFirstX = prev != null && word.getX1() < minX;
boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1(); boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
boolean splittedByRuling = boolean splittedByRuling = isSplittedByRuling(maxX,
isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) || minY,
isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) word.getX1(),
word.getY1(),
verticalRulingLines,
word.getDir().getDegrees(),
cropbox.getWidth(),
cropbox.getHeight()) || isSplittedByRuling(minX,
minY,
word.getX1(),
word.getY2(),
horizontalRulingLines,
word.getDir().getDegrees(),
cropbox.getWidth(),
cropbox.getHeight())
|| isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()) || isSplittedByRuling(maxX,
|| isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines, word.getDir().getDegrees(), cropbox.getWidth(), cropbox.getHeight()); minY,
word.getX1(),
word.getY1(),
horizontalRulingLines,
word.getDir().getDegrees(),
cropbox.getWidth(),
cropbox.getHeight()) || isSplittedByRuling(minX,
minY,
word.getX1(),
word.getY2(),
verticalRulingLines,
word.getDir().getDegrees(),
cropbox.getWidth(),
cropbox.getHeight());
boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir()); boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
if (prev != null && (lineSeparation || startFromTop || xIsBeforeFirstX || splitByX || splitByDir || splittedByRuling)) { if (prev != null && (lineSeparation || startFromTop || xIsBeforeFirstX || splitByX || splitByDir || splittedByRuling)) {
@ -122,8 +144,7 @@ public class BlockificationService {
TextBlock block = (TextBlock) itty.next(); TextBlock block = (TextBlock) itty.next();
if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) { if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
.getMinY()) {
previousLeft.add(block); previousLeft.add(block);
itty.remove(); itty.remove();
continue; continue;
@ -131,8 +152,7 @@ public class BlockificationService {
} }
if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) { if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
.getMinY()) {
previousRight.add(block); previousRight.add(block);
itty.remove(); itty.remove();
continue; continue;
@ -151,10 +171,8 @@ public class BlockificationService {
while (itty.hasNext()) { while (itty.hasNext()) {
TextBlock block = (TextBlock) itty.next(); TextBlock block = (TextBlock) itty.next();
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation() if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
.equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY()) || previous != null && previous previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
.getOrientation()
.equals(Orientation.LEFT) && block.getOrientation()
.equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) { .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
previous.add(block); previous.add(block);
itty.remove(); itty.remove();
@ -193,12 +211,10 @@ public class BlockificationService {
styleFrequencyCounter.add(wordBlock.getFontStyle()); styleFrequencyCounter.add(wordBlock.getFontStyle());
if (textBlock == null) { if (textBlock == null) {
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation());
.getRotation());
} else { } else {
TextBlock spatialEntity = textBlock.union(wordBlock); TextBlock spatialEntity = textBlock.union(wordBlock);
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
.getHeight());
} }
} }
@ -211,19 +227,21 @@ public class BlockificationService {
textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest()); textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest());
} }
if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences() if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getY1(), 3)).collect(toSet()).size() == 1) {
.stream()
.map(t -> round(t.getY1(), 3))
.collect(toSet())
.size() == 1) {
textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getX1)); textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getX1));
} }
return textBlock; return textBlock;
} }
private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, private boolean isSplittedByRuling(float previousX2,
List<Ruling> rulingLines, float rotation, float pageWidth, float pageHeight) { float previousY1,
float currentX1,
float currentY1,
List<Ruling> rulingLines,
float rotation,
float pageWidth,
float pageHeight) {
for (Ruling ruling : rulingLines) { for (Ruling ruling : rulingLines) {
var line = CoordSystemHelper.convertToDirAdj(ruling, rotation, pageWidth, pageHeight); var line = CoordSystemHelper.convertToDirAdj(ruling, rotation, pageWidth, pageHeight);
@ -235,13 +253,7 @@ public class BlockificationService {
} }
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter,
boolean landscape) {
float minX = 10000; float minX = 10000;
float maxX = -100; float maxX = -100;
@ -270,7 +282,7 @@ public class BlockificationService {
if (documentFontSizeCounter.getMostPopular() != null) { if (documentFontSizeCounter.getMostPopular() != null) {
if (textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) { if (textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0){ if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
if (textBlock.getY1() < minX) { if (textBlock.getY1() < minX) {
minX = textBlock.getY1(); minX = textBlock.getY1();
} }
@ -310,7 +322,7 @@ public class BlockificationService {
continue; continue;
} }
for (TextBlock textBlock : cell.getTextBlocks()) { for (TextBlock textBlock : cell.getTextBlocks()) {
if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0){ if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
if (textBlock.getY1() < minX) { if (textBlock.getY1() < minX) {
minX = textBlock.getMinY(); minX = textBlock.getMinY();
} }
@ -342,16 +354,13 @@ public class BlockificationService {
} }
} }
} }
System.out.println("Page: " + page.getPageNumber() + " Landscape: " + page.isLandscape()+ " MinX: " + minX + " MaxX: " + maxX + " MinY: " +minY + " MaxY: " + maxY); System.out.println("Page: " + page.getPageNumber() + " Landscape: " + page.isLandscape() + " MinX: " + minX + " MaxX: " + maxX + " MinY: " + minY + " MaxY: " + maxY);
} }
return new Rectangle(new Point(minX, minY), maxX - minX, maxY - minY, 0); return new Rectangle(new Point(minX, minY), maxX - minX, maxY - minY, 0);
} }
private double round(float value, int decimalPoints) { private double round(float value, int decimalPoints) {
var d = Math.pow(10, decimalPoints); var d = Math.pow(10, decimalPoints);

89
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java
View File

@ -1,5 +1,10 @@
package com.iqser.red.service.redaction.v1.server.classification.service; package com.iqser.red.service.redaction.v1.server.classification.service;
import java.util.List;
import java.util.regex.Pattern;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@ -7,12 +12,9 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils; import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.regex.Pattern;
@Slf4j @Slf4j
@Service @Service
@ -34,25 +36,22 @@ public class ClassificationService {
for (Page page : document.getPages()) { for (Page page : document.getPages()) {
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame; Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) {
if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) { btf = new Rectangle(new Point(btf.getTopLeft().getY(), page.getCropBox().getHeight() - btf.getTopLeft().getX() - btf.getWidth()),
btf = new Rectangle(new Point(btf.getTopLeft().getY(),page.getCropBox().getHeight() - btf.getTopLeft().getX() -btf.getWidth()), btf.getHeight(), btf.getWidth(), 0); btf.getHeight(),
} btf.getWidth(),
else if(page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) { 0);
} else if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0); btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0);
} else if (page.getRotation() == 180) {
btf = new Rectangle(new Point(btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() - btf.getHeight()),
btf.getWidth(),
btf.getHeight(),
0);
} }
else if(page.getRotation() == 180){
btf = new Rectangle(new Point( btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() -btf.getHeight()), btf.getWidth(), btf.getHeight(), 0);
}
page.setBodyTextFrame(btf); page.setBodyTextFrame(btf);
classifyPage(btf, page, document, headlineFontSizes); classifyPage(btf, page, document, headlineFontSizes);
} }
} }
@ -68,10 +67,9 @@ public class ClassificationService {
} }
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
List<Float> headlineFontSizes) {
System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() +" B " + textBlock.getSequences().get(0).getDir()); System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() + " B " + textBlock.getSequences().get(0).getDir());
// if (document.getFontSizeCounter().getMostPopular() == null) { // if (document.getFontSizeCounter().getMostPopular() == null) {
// textBlock.setClassification("Other"); // textBlock.setClassification("Other");
@ -91,19 +89,21 @@ public class ClassificationService {
// .getMostPopular())) // .getMostPopular()))
{ {
textBlock.setClassification("Footer"); textBlock.setClassification("Footer");
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame,
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter() textBlock) && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter() document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
.getMostPopular() || page.getTextBlocks().size() == 1)) { .size() == 1)) {
if (!Pattern.matches("[0-9]+", textBlock.toString())) { if (!Pattern.matches("[0-9]+", textBlock.toString())) {
textBlock.setClassification("Title"); textBlock.setClassification("Title");
} }
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
.getFontSizeCounter() .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle() .getCountPerValue()
.equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
.getFontSizeCounter() .get(0)
.getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { .getTextPositions()
.get(0)
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
for (int i = 1; i <= headlineFontSizes.size(); i++) { for (int i = 1; i <= headlineFontSizes.size(); i++) {
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) { if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
@ -111,28 +111,25 @@ public class ClassificationService {
document.setHeadlines(true); document.setHeadlines(true);
} }
} }
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText() } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle() textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
.equals("bold") && !document.getFontStyleCounter()
.getMostPopular() .getMostPopular()
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
.get(0)
.getTextPositions()
.get(0)
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
textBlock.setClassification("H " + (headlineFontSizes.size() + 1)); textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
document.setHeadlines(true); document.setHeadlines(true);
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
.getFontSizeCounter() .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
.getMostPopular() && textBlock.getMostPopularWordStyle()
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
textBlock.setClassification("TextBlock Bold"); textBlock.setClassification("TextBlock Bold");
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont() } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle() .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
.equals(document.getFontStyleCounter() .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
.getMostPopular()) {
textBlock.setClassification("TextBlock"); textBlock.setClassification("TextBlock");
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
.getFontSizeCounter() .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
.getMostPopular() && textBlock.getMostPopularWordStyle()
.equals("italic") && !document.getFontStyleCounter()
.getMostPopular() .getMostPopular()
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) { .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
textBlock.setClassification("TextBlock Italic"); textBlock.setClassification("TextBlock Italic");

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/CoordSystemHelper.java
View File

@ -12,14 +12,15 @@ import lombok.experimental.UtilityClass;
@UtilityClass @UtilityClass
public class CoordSystemHelper { public class CoordSystemHelper {
public Rectangle convertToDirAdj(Rectangle rectangle, float rotation, float pageWidth, float pageHeight) { public Rectangle convertToDirAdj(Rectangle rectangle, float rotation, float pageWidth, float pageHeight) {
var topLeft = convertPoint(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rotation, pageWidth, pageHeight); var topLeft = convertPoint(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rotation, pageWidth, pageHeight);
var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() , rotation, pageWidth, pageHeight); var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rotation, pageWidth, pageHeight);
return new Rectangle(new Point((float) topLeft.getX(), (float) topLeft.getY()),
return new Rectangle(new Point((float)topLeft.getX(),(float) topLeft.getY()), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()), 0); (float) (bottomRight.getX() - topLeft.getX()),
(float) (bottomRight.getY() - topLeft.getY()),
0);
} }

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java
View File

@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.classification.utils;
import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@UtilityClass @UtilityClass
@SuppressWarnings("all") @SuppressWarnings("all")
public class PositionUtils { public class PositionUtils {
public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) { public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
//TODO Currently this is not working for rotated pages. //TODO Currently this is not working for rotated pages.
@ -19,10 +19,9 @@ public class PositionUtils {
double threshold = textBlock.getMostPopularWordHeight() * 3; double threshold = textBlock.getMostPopularWordHeight() * 3;
if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() && if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() && textBlock.getMaxX() - threshold < btf.getTopLeft()
textBlock.getMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth() && .getX() + btf.getWidth() && textBlock.getMinY() + threshold > btf.getTopLeft().getY() && textBlock.getMaxY() - threshold < btf.getTopLeft()
textBlock.getMinY() + threshold > btf.getTopLeft().getY() && .getY() + btf.getHeight()) {
textBlock.getMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
return true; return true;
} else { } else {
return false; return false;
@ -41,7 +40,7 @@ public class PositionUtils {
return true; return true;
} }
if(rotation == 180 && textBlock.getY2() < btf.getTopLeft().getY()){ if (rotation == 180 && textBlock.getY2() < btf.getTopLeft().getY()) {
return true; return true;
} }
@ -60,7 +59,6 @@ public class PositionUtils {
public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) { public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
if (btf == null || textBlock == null) { if (btf == null || textBlock == null) {
return false; return false;
} }
@ -104,11 +102,14 @@ public class PositionUtils {
public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) { public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) {
return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight; return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
} }
public Float getApproxLineCount(TextBlock textBlock) { public Float getApproxLineCount(TextBlock textBlock) {
return textBlock.getHeight() / textBlock.getMostPopularWordHeight(); return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
} }
} }

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java
View File

@ -147,7 +147,6 @@ public class TextPositionSequence implements CharSequence {
} }
@JsonIgnore @JsonIgnore
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getRotationAdjustedY() { public float getRotationAdjustedY() {
@ -160,7 +159,6 @@ public class TextPositionSequence implements CharSequence {
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getX1() { public float getX1() {
return textPositions.get(0).getXDirAdj(); return textPositions.get(0).getXDirAdj();
} }
@ -170,24 +168,15 @@ public class TextPositionSequence implements CharSequence {
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getX2() { public float getX2() {
return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING; return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
} }
@JsonIgnore @JsonIgnore
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getY1() { public float getY1() {
return textPositions.get(0).getYDirAdj() - getTextHeight(); return textPositions.get(0).getYDirAdj() - getTextHeight();
} }
@ -197,15 +186,11 @@ public class TextPositionSequence implements CharSequence {
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getY2() { public float getY2() {
return textPositions.get(0).getYDirAdj(); return textPositions.get(0).getYDirAdj();
} }
@JsonIgnore @JsonIgnore
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getTextHeight() { public float getTextHeight() {

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -101,18 +101,20 @@ public class PdfSegmentationService {
PDRectangle pdr = pdPage.getMediaBox(); PDRectangle pdr = pdPage.getMediaBox();
int rotation = pdPage.getRotation(); int rotation = pdPage.getRotation();
boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
PDRectangle cropbox = pdPage.getCropBox(); PDRectangle cropbox = pdPage.getCropBox();
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight()); CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
stripper.getRulings(),
stripper.getMinCharWidth(),
stripper.getMaxCharHeight());
Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), rotation, cropbox); Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), rotation, cropbox);
float cropboxArea = cropbox.getHeight() * cropbox.getWidth(); float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
page.setCropBoxArea(cropboxArea); page.setCropBoxArea(cropboxArea);
System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" +cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox().getWidth() + " h:" +pdPage.getMediaBox().getHeight()); System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" + cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox()
.getWidth() + " h:" + pdPage.getMediaBox().getHeight());
page.setRotation(rotation); page.setRotation(rotation);
page.setLandscape(isLandscape); page.setLandscape(isLandscape);

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java
View File

@ -31,31 +31,41 @@ public abstract class AbstractTextContainer {
@JsonIgnore @JsonIgnore
private Orientation orientation = Orientation.NONE; private Orientation orientation = Orientation.NONE;
public abstract String getText(); public abstract String getText();
public boolean containsBlock(TextBlock other) { public boolean containsBlock(TextBlock other) {
return this.minX <= other.getX1() && this.maxX >= other.getX2() && this.minY >= other.getY1() && this.maxY <= other.getY2(); return this.minX <= other.getX1() && this.maxX >= other.getX2() && this.minY >= other.getY1() && this.maxY <= other.getY2();
} }
public boolean contains(AbstractTextContainer other) { public boolean contains(AbstractTextContainer other) {
return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY; return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
} }
public boolean contains(Rectangle other) { public boolean contains(Rectangle other) {
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft()
.getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
} }
@JsonIgnore @JsonIgnore
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getHeight() { public float getHeight() {
return maxY - minY; return maxY - minY;
} }
@JsonIgnore @JsonIgnore
@JsonAttribute(ignore = true) @JsonAttribute(ignore = true)
public float getWidth() { public float getWidth() {
return maxX - minX; return maxX - minX;
} }

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java
View File

@ -1,15 +1,27 @@
package com.iqser.red.service.redaction.v1.server.tableextraction.service; package com.iqser.red.service.redaction.v1.server.tableextraction.service;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import java.awt.geom.Point2D;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import java.util.ArrayList;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.*; import java.util.Comparator;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.awt.geom.Point2D; import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import java.util.*; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import java.util.stream.Collectors; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
@Service @Service
public class TableExtractionService { public class TableExtractionService {

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
View File

@ -1,5 +1,15 @@
package com.iqser.red.service.redaction.v1.server.visualization.service; package com.iqser.red.service.redaction.v1.server.visualization.service;
import java.awt.Color;
import java.io.IOException;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.Page;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph; import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
@ -7,24 +17,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDPageContentStream;
import org.apache.pdfbox.pdmodel.font.PDType1Font;
import org.springframework.stereotype.Service;
import java.awt.Color;
import java.io.IOException;
import java.util.List;
@Slf4j @Slf4j
@Service @Service
@RequiredArgsConstructor @RequiredArgsConstructor
public class PdfVisualisationService { public class PdfVisualisationService {
public void visualizeParagraphs(Document classifiedDoc, PDDocument document) throws IOException { public void visualizeParagraphs(Document classifiedDoc, PDDocument document) throws IOException {
for (int page = 1; page <= document.getNumberOfPages(); page++) { for (int page = 1; page <= document.getNumberOfPages(); page++) {
@ -66,7 +67,6 @@ public class PdfVisualisationService {
PDPage pdPage = document.getPage(page - 1); PDPage pdPage = document.getPage(page - 1);
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true); PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
for (AbstractTextContainer textBlock : analyzedPage.getTextBlocks()) { for (AbstractTextContainer textBlock : analyzedPage.getTextBlocks()) {
if (textBlock == null) { if (textBlock == null) {
continue; continue;
@ -84,7 +84,10 @@ public class PdfVisualisationService {
// //
// contentStream.addRect(analyzedPage.getBodyTextFrame().getX1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), analyzedPage.getBodyTextFrame().getY1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), width, height); // contentStream.addRect(analyzedPage.getBodyTextFrame().getX1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), analyzedPage.getBodyTextFrame().getY1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), width, height);
contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), analyzedPage.getBodyTextFrame().getTopLeft().getY(), analyzedPage.getBodyTextFrame().getWidth(), analyzedPage.getBodyTextFrame().getHeight()); contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(),
analyzedPage.getBodyTextFrame().getTopLeft().getY(),
analyzedPage.getBodyTextFrame().getWidth(),
analyzedPage.getBodyTextFrame().getHeight());
contentStream.stroke(); contentStream.stroke();
@ -111,7 +114,6 @@ public class PdfVisualisationService {
contentStream.endText(); contentStream.endText();
// contentStream.setNonStrokingColor(Color.BLUE); // contentStream.setNonStrokingColor(Color.BLUE);
// contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f); // contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f);
// //
@ -140,6 +142,7 @@ public class PdfVisualisationService {
private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException { private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
for (List<Cell> row : table.getRows()) { for (List<Cell> row : table.getRows()) {
for (Cell cell : row) { for (Cell cell : row) {
@ -170,4 +173,5 @@ public class PdfVisualisationService {
contentStream.endText(); contentStream.endText();
} }
} }
} }